C#模拟用户登录，数据抓取包括分页爬取插入数据…
2018-06-22 07:57:55来源：未知 阅读 ()
1 using System; 2 using System.Collections.Generic; 3 using System.Web; 4 using System.Web.Services; 5 using Topevery.DUM.GridingManagement.WWFLocalHelper.WorkflowHelper; 6 using Topevery.DUM.GridingManagement.Entity.Event; 7 using Topevery.DUM.GridingManagement.ConfigHelper; 8 using Topevery.Workflow.ObjectModel.StateMachineWorkflow; 9 using Topevery.DUM.GridingManagement.Logic; 10 using Topevery.DUM.GridingManagement.Entity; 11 using Topevery.DUM.GridingManagement.Framework.Utility; 12 using Topevery.DUM.GridingManagement.Entity.Enum; 13 using System.Data; 14 15 namespace Topevery.DUM.GridingManagement.Web.ASMX 16 { 17 /// <summary> 18 /// _2345CrawlAdd 的摘要说明 19 /// </summary> 20 [WebService(Namespace = "http://localhost/s")] 21 [WebServiceBinding(ConformsTo = WsiProfiles.BasicProfile1_1)] 22 [System.ComponentModel.ToolboxItem(false)] 23 24 25 26 public class _2345CrawlAdd : System.Web.Services.WebService 27 { 28 [WebMethod(Description = @"12345网站抓取数据参数汇总nums0/案件提交用户 29 ,nums1/案件举报时间,nums2/举报电话,nums3/举报电话1, 30 nums4/举报电话2,nums5/案件举报时间,nums6/案件举报时间, 31 nums7/12345案件号,nums8/举报电话,nums9/备注")] 32 33 34 35 public void StartEvents(string nums0, string nums1, string nums2, string nums3, string nums4 36 , string nums5, string nums6, string nums7, string nums8, string nums9, string nums10) 37 { 38 DataTable dtEvent = UmEventLogic.GetEvent_id(); 39 if (dtEvent.Select(string.Format("c_event_id={0}", nums7.Trim())).Length > 0) 40 { 41 LogHelper.Log.Debug(string.Format("事项编号：{0}的案件已存在。", nums7)); 42 return; 43 } 44 string Reporter = nums0; 45 string EventDesc = nums9; 46 string datetime = nums1; 47 string TelNum = nums2; 48 string[] time = nums2.Split(','); 49 string phone0 = "", phone1 = "", phone2 = "", phone3 = ""; 50 if (time.Length > 0) 51 { 52 phone0 = time[0]; 53 phone1 = time[1]; 54 phone2 = time[2]; 55 phone3 = time[3]; 56 if (phone2 == "无") 57 { 58 phone2 = "无联系电话2"; 59 } 60 else 61 { 62 phone2 = time[2]; 63 } 64 } 65 UmEvent evt = new UmEvent(); 66 evt.ID = Guid.NewGuid(); 67 //evt.WfInstID = new Guid("5C02AAD4-58F2-45A6-8551-8C940583A210"); 68 evt.IsReceipt = false; 69 evt.ReplyWayID = new Guid("671aabcf-592f-462e-b395-064105c8f391"); 70 evt.ReplyWay = time[0]; 71 evt.DistrictID = new Guid("a4879be6-5746-4fcf-8fa7-49eefa1d656e"); 72 evt.StreetID = new Guid("edf2d849-a526-4165-aeb2-a776de6e53ad"); 73 evt.CommunityID = null; 74 evt.WorkGridID = null; 75 evt.GridID = null; 76 evt.TypeID = new Guid("abff5920-9838-4fdc-ab3b-1dfe773991da"); 77 evt.BigClassID = new Guid("ae217b5d-e9b6-dd11-8009-001e90a20a0c"); 78 evt.SmallClassID = new Guid("c8217b5d-e9b6-dd11-8009-001e90a20a0c"); 79 evt.ImportantLevel = EventLevelEnum.Level0; 80 //evt.KeyWord = nums7; 81 evt.KeyWord = "1"; 82 evt.Position = "12345转办"; 83 DateTime fromTime=DateTime.Now; 84 string hhMMss = datetime.Substring(datetime.Length - 8, 8); 85 if (Convert.ToInt32(hhMMss.Substring(0, 1)) > 2 || Convert.ToInt32(hhMMss.Substring(0, 1)) == 2) 86 { 87 fromTime = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd") + " 0" + datetime.Substring(datetime.Length - 7, 7)); 88 } 89 else 90 { 91 fromTime = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd") + " " + hhMMss); 92 } 93 94 evt.EventDesc = "【案件内容】" + nums9 + "【来件时间】" + fromTime; 95 evt.Reporter = nums0;//来件人 96 //evt.Status = 0; 97 evt.Summary = "【事项编号】" + nums7 + "【坐席编号】" + nums8 + "【联系电话1】" + phone1 + "【联系电话2】" + phone2 + "【联系电话3】" + time[3] + "【先前回复】" + nums10; 98 evt.TelNum = time[0]; 99 evt.DbLastUpdateDate = DateTime.Now; 100 evt.DbLastUpdateID = new Guid("56a90ca0-a14f-45fc-8280-a7a0935271d7"); 101 evt.EvtSrID = new Guid("BCE7ABB4-99BE-431A-8744-3812403D3227"); 103 evt.DbCreateID = new Guid("56a90ca0-a14f-45fc-8280-a7a0935271d7"); 104 evt.DbCreateDate = DateTime.Now; 105 106 try 107 { 108 UmEventLogic.SaveUmEvent(evt);//添加到案件主表 109 StartEvent(evt); 110 UmEventLogic.SaveUmEvents(evt.ID, nums7);//添加和12345关联的案件ID 111 } 112 catch (Exception ex) 113 { 114 LogHelper.Log.Error("添加案件主表异常：" + ex); 115 } 116 117 } 118 119 #region 方法重载 12345接口添加新的重载方法 120 [WebMethod(Description = @"12345网站抓取数据参数汇总nums0/案件提交用户 121 ,nums1/案件举报时间,nums2/举报电话,nums3/举报电话1, 122 nums4/举报电话2,nums5/案件举报时间,nums6/案件举报时间, 123 nums7/12345案件号,nums8/举报电话,nums9/(此处应为内容)备注,TargetId/目标环节的办理者Id,DistrictID/区域ID,StreetID/街道ID")] 124 125 public void StartEvents_New(string nums0, string nums1, string nums2, string nums3, string nums4 126 , string nums5, string nums6, string nums7, string nums8, string nums9, string nums10, Guid TargetId, Guid DistrictID, Guid StreetID) 127 { 128 DataTable dtEvent = UmEventLogic.GetEvent_id(); 129 if (dtEvent.Select(string.Format("c_event_id={0}", nums7.Trim())).Length > 0) 130 { 131 LogHelper.Log.Debug(string.Format("事项编号：{0}的案件已存在。", nums7)); 132 return; 133 } 134 string Reporter = nums0; 135 string EventDesc = nums9; 136 string datetime = nums1; 137 138 string phone0 = "", phone1 = "", phone2 = "", phone3 = ""; 139 phone0 = nums2; 140 phone1 = nums3; 141 phone2 = nums4; 160 UmEvent evt = new UmEvent(); 161 evt.ID = Guid.NewGuid(); 163 evt.IsReceipt = false; 164 evt.ReplyWayID = new Guid("671aabcf-592f-462e-b395-064105c8f391"); 166 evt.ReplyWay = nums2; 167 evt.DistrictID = DistrictID; //区域ID 168 evt.StreetID = StreetID; //街道ID 169 evt.CommunityID = null; 170 evt.WorkGridID = null; 171 evt.GridID = null; 175 BigSmallClass entity = BigSmallClassLogic.GetBigSmallClassEntity(new Guid("BE9B75EA-D602-4EC1-BC98-CDF55DB01F54")); 176 if (entity != null) 177 { 178 evt.TypeID = entity.EvtTypeID; 179 evt.BigClassID = entity.BigClassID; 180 evt.SmallClassID = entity.SmallClassID; 181 } 182 evt.ImportantLevel = EventLevelEnum.Level0; 183 evt.KeyWord = "1"; 184 evt.Position = "12345转办"; 185 evt.KeyWord = "【事项编号】" + nums7; 186 DateTime fromTime = DateTime.Now; 187 string hhMMss = string.Empty; 188 189 if (!string.IsNullOrEmpty(datetime)) 190 { 191 hhMMss = datetime.Substring(datetime.Length - 8, 8); 192 } 193 if (!string.IsNullOrEmpty(hhMMss) && (Convert.ToInt32(hhMMss.Substring(0, 1)) > 2 || Convert.ToInt32(hhMMss.Substring(0, 1)) == 2)) 194 { 195 fromTime = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd") + " 0" + datetime.Substring(datetime.Length - 7, 7)); 196 } 197 else 198 { 199 fromTime = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd") + " " + hhMMss); 200 } 201 202 203 string addressStr = string.Empty; 204 string EventDescStr = string.Empty; 212 #region 213 //string fromTime = "2012-10-08"; 214 //判断是否有事项一 215 int sxIndexOf; 216 if (nums9.IndexOf("事项：") != -1) //存在事项一 217 { 218 sxIndexOf = nums9.IndexOf("事项："); 219 //判断是否有地址 220 int addIndexOf1 = nums9.IndexOf("地址一："); 221 int addIndexOf2 = nums9.IndexOf("地址二："); 222 int addIndexOf3 = nums9.IndexOf("地址三："); 223 if (addIndexOf3 != -1) 224 { 225 addressStr = nums9.Substring(addIndexOf1, addIndexOf2) + "；" + nums9.Substring(addIndexOf2, addIndexOf3 - addIndexOf2) + "；" + nums9.Substring(addIndexOf3, sxIndexOf - addIndexOf3); 226 EventDescStr = "【案件内容】" + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 227 } 228 else if (addIndexOf2 != -1) 229 { 230 addressStr = nums9.Substring(addIndexOf1, addIndexOf2 - addIndexOf1) + "；" + nums9.Substring(addIndexOf2, sxIndexOf - addIndexOf2); 231 EventDescStr = "【案件内容】" + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 232 } 233 else 234 { 235 if (string.Compare(nums9.Substring(0, 2), "地址", true) == 0) //有地址存在 236 { 237 if (string.Compare(nums9.Substring(0, 3), "地址：", true) == 0) //只有一个地址 238 { 239 int addressIndexOf0 = nums9.IndexOf("地址："); 240 addressStr = nums9.Substring(addressIndexOf0, sxIndexOf); 241 EventDescStr = "【案件内容】" + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 242 } 243 } 244 else 245 { 246 int addressIndexOf0 = nums9.IndexOf("地址："); 247 if (addressIndexOf0 == -1) 248 { 249 addressStr = ""; 250 //addressStr = "暂无地址"; 251 EventDescStr = "【案件内容】" + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 252 } 253 else 254 { 255 addressStr = nums9.Substring(addressIndexOf0, sxIndexOf - addressIndexOf0); 256 EventDescStr = "【案件内容】" + nums9.Substring(0, addressIndexOf0) + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 257 } 258 } 259 } 260 } 261 else if (nums9.IndexOf("事项:") != -1) //存在事项一 262 { 263 sxIndexOf = nums9.IndexOf("事项:"); 264 //判断是否有地址 265 int addIndexOf1 = nums9.IndexOf("地址一："); 266 int addIndexOf2 = nums9.IndexOf("地址二："); 267 int addIndexOf3 = nums9.IndexOf("地址三："); 268 if (addIndexOf3 != -1) 269 { 270 addressStr = nums9.Substring(addIndexOf1, addIndexOf2) + "；" + nums9.Substring(addIndexOf2, addIndexOf3 - addIndexOf2) + "；" + nums9.Substring(addIndexOf3, sxIndexOf - addIndexOf3); 271 EventDescStr = "【案件内容】" + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 272 } 273 else if (addIndexOf2 != -1) 274 { 275 addressStr = nums9.Substring(addIndexOf1, addIndexOf2) + "；" + nums9.Substring(addIndexOf2, sxIndexOf - addIndexOf2); 276 EventDescStr = "【案件内容】" + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 277 } 278 else 279 { 280 if (string.Compare(nums9.Substring(0, 2), "地址", true) == 0) //有地址存在 281 { 282 if (string.Compare(nums9.Substring(0, 3), "地址：", true) == 0) //只有一个地址 283 { 284 int addressIndexOf0 = nums9.IndexOf("地址："); 285 addressStr = nums9.Substring(addressIndexOf0, sxIndexOf); 286 EventDescStr = "【案件内容】" + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 287 } 288 } 289 else 290 { 291 int addressIndexOf0 = nums9.IndexOf("地址："); 292 if (addressIndexOf0 == -1) 293 { 294 addressStr = ""; 295 //addressStr = "暂无地址"; 296 EventDescStr = "【案件内容】" + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 297 } 298 else 299 { 300 addressStr = nums9.Substring(addressIndexOf0, sxIndexOf - addressIndexOf0); 301 EventDescStr = "【案件内容】" + nums9.Substring(0, addressIndexOf0) + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 302 } 303 } 304 } 305 } 306 else if (nums9.IndexOf("事项一：") != -1) 307 { 308 if (nums9.IndexOf("事项三：") != -1 && nums9.IndexOf("地址三：") != -1) //事件三存在 309 { 310 int addIndexOf1 = nums9.IndexOf("地址一："); 311 int addIndexOf2 = nums9.IndexOf("地址二："); 312 int addIndexOf3 = nums9.IndexOf("地址三："); 313 int sjIndexOf1 = nums9.IndexOf("事项一："); 314 int sjIndexOf2 = nums9.IndexOf("事项二："); 315 int sjIndexOf3 = nums9.IndexOf("事项三："); 316 addressStr = nums9.Substring(addIndexOf1, sjIndexOf1 - addIndexOf1) + "；" + nums9.Substring(addIndexOf2, sjIndexOf2 - addIndexOf2) + "；" + nums9.Substring(addIndexOf3, sjIndexOf3 - addIndexOf3); 317 EventDescStr = "【案件内容】" + nums9.Substring(0, addIndexOf1) + nums9.Substring(sjIndexOf1, addIndexOf2 - sjIndexOf1) + "；" + nums9.Substring(sjIndexOf2, addIndexOf3 - sjIndexOf2) + "；" + nums9.Substring(sjIndexOf3) + "【来件时间】" + fromTime; 318 } 319 else 320 { 321 if (nums9.IndexOf("事项二：") != -1 && nums9.IndexOf("地址二：") != -1) //事件二存在 322 { 323 int addIndexOf1 = nums9.IndexOf("地址一："); 324 int addIndexOf2 = nums9.IndexOf("地址二："); 325 int sjIndexOf1 = nums9.IndexOf("事项一："); 326 int sjIndexOf2 = nums9.IndexOf("事项二："); 327 addressStr = nums9.Substring(addIndexOf1, sjIndexOf1 - addIndexOf1) + "；" + nums9.Substring(addIndexOf2, sjIndexOf2 - addIndexOf2); 328 EventDescStr = "【案件内容】" + nums9.Substring(0, addIndexOf1) + nums9.Substring(sjIndexOf1, addIndexOf2 - sjIndexOf1) + "；" + nums9.Substring(sjIndexOf2) + "【来件时间】" + fromTime; 329 } 330 else if (nums9.IndexOf("地址：") != -1 && nums9.IndexOf("地址一：") == -1) 331 { 332 int addIndexOf = nums9.IndexOf("地址："); 333 int sjIndexOf1 = nums9.IndexOf("事项一："); 334 addressStr = nums9.Substring(addIndexOf, sjIndexOf1 - addIndexOf); 335 EventDescStr = "【案件内容】" + nums9.Substring(0, addIndexOf) + nums9.Substring(sjIndexOf1) + "【来件时间】" + fromTime; 336 } 337 } 338 } 339 else 340 { 341 sxIndexOf = nums9.IndexOf("副项："); 342 //判断是否有地址 343 if (string.Compare(nums9.Substring(0, 2), "地址", true) == 0) //有地址存在 344 { 345 if (string.Compare(nums9.Substring(0, 3), "地址：", true) == 0) //只有一个地址 346 { 347 int addressIndexOf0 = nums9.IndexOf("地址："); 348 addressStr = nums9.Substring(addressIndexOf0, sxIndexOf); 349 EventDescStr = "【案件内容】" + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 350 } 351 } 352 else if (sxIndexOf != -1) 353 { 354 int addressIndexOf0 = nums9.IndexOf("地址："); 355 addressStr = nums9.Substring(addressIndexOf0, sxIndexOf - addressIndexOf0); 356 EventDescStr = "【案件内容】" + nums9.Substring(0, addressIndexOf0) + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 357 } 358 else 359 { 360 sxIndexOf = nums9.IndexOf("事项；"); 361 if (sxIndexOf == -1) 362 { 363 sxIndexOf = nums9.IndexOf("事项："); 364 } 365 int addressIndexOf0 = nums9.IndexOf("地址："); 366 if (sxIndexOf > addressIndexOf0) 367 { 368 addressStr = nums9.Substring(addressIndexOf0, sxIndexOf - addressIndexOf0); 369 } 370 else 371 { 372 addressStr = nums9.Substring(addressIndexOf0, addressIndexOf0 - sxIndexOf); 373 } 374 EventDescStr = "【案件内容】" + nums9.Substring(0, addressIndexOf0) + nums9.Substring(sxIndexOf) + "【来件时间】" + fromTime; 375 } 376 } 377 if (nums9.IndexOf("地址：") == -1 && nums9.IndexOf("地址一：") == -1) //没有地址 378 { 379 addressStr = string.Empty; 380 EventDescStr = "【案件内容】" + nums9 + "【来件时间】" + fromTime; 381 } 382 #endregion 383 evt.Position = addressStr; 384 evt.EventDesc = EventDescStr; 385 evt.Reporter = nums0;//来件人 386 //evt.Status = 0; 387 // evt.Summary = "【事项编号】" + nums7 + "【坐席编号】" + nums8 + "【联系电话1】" + phone1 + "【联系电话2】" + phone2 + "【联系电话3】" + time[3] + "【先前回复】" + nums10; 388 evt.Summary = "【事项编号】" + nums7 + "【坐席编号】" + nums8 + "【联系电话1】" + phone1 + "【联系电话2】" + phone2 + "【联系电话3】" + phone3 + "【先前回复】" + nums10; 389 //evt.TelNum = time[0]; 390 evt.TelNum = nums2; 391 evt.DbLastUpdateDate = DateTime.Now; 392 evt.DbLastUpdateID = new Guid("56a90ca0-a14f-45fc-8280-a7a0935271d7"); 393 evt.EvtSrID = new Guid("BCE7ABB4-99BE-431A-8744-3812403D3227");//江门在用的BCE7ABB4-99BE-431A-8744-3812403D3227 394 //本地在用的AA97039F-3CDD-4ED7-8E5C-02863235A239 395 evt.DbCreateID = new Guid("56a90ca0-a14f-45fc-8280-a7a0935271d7"); 396 evt.DbCreateDate = DateTime.Now; 397 398 try 399 { 400 UmEventLogic.SaveUmEvent(evt);//添加到案件主表 401 UmDeptProc deptProcentity = new UmDeptProc(); 402 deptProcentity.EvtID = evt.ID; 403 deptProcentity.RootDeptID = TargetId; 404 deptProcentity.DeptID = TargetId; 405 deptProcentity.Protype = 0; 406 deptProcentity.Promin = 120; 407 deptProcentity.BeginDate = DateTime.Now; 408 deptProcentity.EndDate = DateTime.Now; 409 deptProcentity.SelfPromin =120; 410 deptProcentity.AllPromin = 120; 411 deptProcentity.DeptLevel = 0; 412 UmEventLogic.DeptProcInsert(deptProcentity); 413 StartEvent(evt, TargetId); 414 UmEventLogic.SaveUmEvents(evt.ID, nums7); 415 } 416 catch (Exception ex) 417 { 418 LogHelper.Log.Error("添加案件主表异常：" + ex); 419 } 420 } 421 422 public void StartEvent(UmEvent evt, Guid TargetId) 423 { 424 List<Guid> roleIds = WorkflowHelper.GetEvtWorkflowStartRoleIds(); 425 Guid user_id = evt.DbCreateID; 426 //启动工作流 427 Guid workflowInstanceId = WorkflowHelper.StartEvtWorkflow(evt.ID, user_id, roleIds[0]); 428 evt.WfInstID = workflowInstanceId; 429 //evt.Status = 0; 430 UmEventLogic.UpdateUmEventEntity(evt, null); 431 432 WorkflowHelper.SendEvtToSpecialActivity(evt.ID, evt.WfInstID.Value, user_id, new Guid("1666A3B8-D403-4553-AF32-4D2E253E2F03"), 433 TargetId, Guid.Empty, Topevery.Workflow.ObjectModel.ActivitySendTargetType.Department, "请处理"); 434 435 } 436 437 #endregion 438 439 [WebMethod(Description = "获取12345案件ID")] 440 public void GetEvent_ids() 441 { 442 UmEventLogic.SaveUmEvents(new Guid("4BDA095A-5134-487C-8F75-E8FB47FA59D9"), "363636"); 443 } 444 [WebMethod(Description = "获取12345案件ID")] 445 public DataTable GetEvent_id() 446 { 447 return UmEventLogic.GetEvent_id(); 448 } 449 450 public void StartEvent(UmEvent evt) 451 { 452 List<Guid> roleIds = WorkflowHelper.GetEvtWorkflowStartRoleIds(); 453 Guid user_id = evt.DbCreateID; 454 //启动工作流 455 Guid workflowInstanceId = WorkflowHelper.StartEvtWorkflow(evt.ID, user_id, roleIds[0]); 456 evt.WfInstID = workflowInstanceId; 457 //evt.Status = 0; 458 UmEventLogic.UpdateUmEventEntity(evt, null); 459 460 WorkflowHelper.SendEvtToSpecialActivity(evt.ID, evt.WfInstID.Value, user_id, new Guid("1666A3B8-D403-4553-AF32-4D2E253E2F03"), 461 new Guid("6e255837-c8ff-44f7-ab7d-7373c325e54a"), Guid.Empty, Topevery.Workflow.ObjectModel.ActivitySendTargetType.Department, "请处理"); 462 463 } 464 List<Guid> FilterRoleIds(List<Guid> roleIds, UmEvent evt, Guid obId) 465 { 466 Map map = Topevery.DUM.GridingManagement.Logic.Utility.GetEvtMinRegion(evt); 467 return Logic.Utility.FilterRoleTarget(roleIds, map); 468 } 469 void SaveUmObEvtTaskErr(int taskId, string message) 470 { 471 LogHelper.Log.Error(string.Format("taskId:{0},错误:{1}", taskId, message)); 472 } 473 } 474 }
//涉及到的dll，Winista.HtmlParser 获取表单数据
using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Net; using System.IO; using Topevery.DUM.Reported.Entity; using Topevery.DUM.ReportedService.Code; using System.Data; using Topevery.DUM.GridingManagement.Entity.Event; using Topevery.DUM.GridingManagement.Logic; using Topevery.DUM.GridingManagement.Entity; using Topevery.Framework.MonitorServices; using Topevery.DUM.ZhaQuServer._12345CrawlAdd; using System.Diagnostics; using Winista.Text.HtmlParser; using Winista.Text.HtmlParser.Data; using Winista.Text.HtmlParser.Extractors; using Winista.Text.HtmlParser.Filters; using Winista.Text.HtmlParser.Http; using Winista.Text.HtmlParser.Lex; using Winista.Text.HtmlParser.Nodes; using Winista.Text.HtmlParser.Scanners; using Winista.Text.HtmlParser.Support; using Winista.Text.HtmlParser.Tags; using Winista.Text.HtmlParser.Util; using Winista.Text.HtmlParser.Visitors; namespace Topevery.DUM.ZhaQuServer.Code { public class HtmlPagerParser { // Fields private _2345CrawlAdd _12345; private CookieContainer cc; string lastKey = ""; Dictionary<string, string> dicList = new Dictionary<string, string>();//存放事件编号与所属流程等信息 // Methods public HtmlPagerParser() { this.cc = new CookieContainer(); this._12345 = new _2345CrawlAdd(); try { this.Login(); } catch (Exception exception) { LogHelper.Log.Error(exception); } } public HtmlPagerParser(string login_name, string pwss_word) { this.cc = new CookieContainer(); this._12345 = new _2345CrawlAdd(); try { this.Login(login_name, pwss_word); LogHelper.Log.Debug(string.Format("用户：{0} 登录成功。", login_name)); } catch (Exception exception) { LogHelper.Log.Error(exception); } } private AcceptParameter BuildEntityFrom12345(EventInPara para) { AcceptParameter parameter = new AcceptParameter(); parameter.Desc = string.Format("{0} {1}", para.EventTitle, para.EventDescript); parameter.Reporter = para.ClientName; parameter.TelNum = string.Format("{0},{1}", para.Phone1, para.Phone2); parameter.DbCreateDate = new DateTime?(para.EventProcessTime); parameter.ReplyType = 1; parameter.Event_id = para.EventID.ToString(); parameter.ReplyWay = para.Phone1; return parameter; } private string[] Get12345ContentDetail(string id) { try { string strResult = ReturnHtml(id); if (!string.IsNullOrEmpty(strResult)) { try { Regex tiqu = new Regex("(?<=<table class='moform'[^>]*>)[\\s\\S]*?(?=</table>)"); MatchCollection mc = tiqu.Matches(strResult, 0); string[] num = new string[mc.Count]; for (int i = 0; i < mc.Count; i++) { num[i] = mc[i].ToString(); } //---获取事件详细信息--- //string a = num[1]; string a = num[0]; Regex tiqus = new Regex("(?<=<tr[^>]*>)[\\s\\S]*?(?=</tr>)"); MatchCollection mcs = tiqus.Matches(a, 0); string[] sb = new string[mcs.Count + 5]; for (int i = 0; i < mcs.Count + 3; i++) { string bc = Regex.Replace(a, "(<.+?>)|(\\s*)|( )", "", RegexOptions.IgnoreCase); string Des = Regex.Replace(num[2], "(<.+?>)|(\\s*)|( )", "", RegexOptions.IgnoreCase); string Reply = Regex.Replace(num[4], "(<.+?>)|(\\s*)|( )", "", RegexOptions.IgnoreCase); string replyName = Reply.Split(';')[3].ToString().Split('】')[3].ToString(); int fwdx = bc.IndexOf("服务对象姓名 ") + 12; int numName = bc.IndexOf("转办时间 ") + 10; int numPhone1 = bc.IndexOf("联系电话1 ") + 11; int numPhone2 = bc.IndexOf("联系电话2 ") + 11; int numPhone3 = bc.IndexOf("联系电话3 ") + 11; int yx = bc.IndexOf(" 邮箱地址") + 10; int numEventID = bc.IndexOf("工单编号 ") + 10; int numAgentID = bc.IndexOf("服务人员工号 ") + 12; int numTitle = Des.IndexOf("事项标题 "); int xxnr = Des.IndexOf("事项内容 ") + 10; int sxfj = Des.IndexOf("事项附件") + 4; switch (i) { case 0: //案件提交用户 --服务对象姓名 sb[i] = bc.Substring(fwdx, numName - fwdx - 10); break; case 1: //案件举报时间; --转办时间 sb[i] = bc.Substring(numName, numPhone1 - numName - 11); break; case 2: //联系电话1 sb[i] = bc.Substring(numPhone1, numPhone2 - numPhone1 - 11); string[] str = bc.Substring(42, 60).Split(';'); string time1 = str[1].Substring(0, str[1].IndexOf("联系电话")); break; case 3: //举报电话2 sb[i] = bc.Substring(numPhone2, numPhone3 - numPhone2 - 11); str = bc.Substring(42, 60).Split(';'); string time2 = str[2].Substring(0, str[2].IndexOf("联")); if (time2 == "无联系电话3 ") { time2 = null; } //sb[i] = time2; break; case 4: //举报电话3 int index = 0; sb[i] = bc.Substring(numPhone3, yx - numPhone3 - 10); str = bc.Substring(42, 60).Split(';'); string time3 = string.Empty; index = str[3].IndexOf("&"); if (index != -1) { time3 = str[3].Substring(0, index); } break; case 5: //案件举报时间 --转办时间 sb[i] = bc.Substring(numName, numPhone1 - numName - 11); break; case 6: //案件举报时间 --转办时间 sb[i] = bc.Substring(numName, numPhone1 - numName - 11); break; case 7: //12345案件号 sb[i] = bc.Substring(numEventID, numAgentID - numEventID - 12); break; case 8: //举报电话 sb[i] = bc.Substring(numPhone1, numPhone2 - numPhone1 - 11); //(此处应为事项内容)备注 sb[9] = Des.Substring(xxnr, sxfj - xxnr - 4); sb[10] = replyName + "!"; break; } } return sb; } catch (Exception ex) { LogHelper.Log.Error(ex.Message, ex); } } } catch (Exception ex) { LogHelper.Log.Error(ex.Message, ex); } return null; } public void Get12345Data() { Dictionary<string,string> dicList = this.Get12345ProjectCodes(); List<string> ids=dicList.Keys.ToList<string>(); List<string> ids2 = new List<string>(); DataTable table = new DataTable(); table = this._12345.GetEvent_id(); if (ids != null) { for (int i = 0; i < ids.Count; i++) { if (!string.IsNullOrEmpty(ids[i])) { if (table.Select(string.Format("c_event_id='{0}'", ids[i])).Length > 0) { LogHelper.Log.Debug(string.Format("案件ID{0}的案件已存在系统中",ids[i])); } else { ids2.Add(ids[i]); } } } for (int i = 0; i < ids2.Count; i++) { if (ids2[i] != null) { try { string[] strArray3 = this.Get12345ContentDetail(ids2[i]); LogHelper.Log.Info("获取内容页数据成功！"); if (strArray3 != null) { LogHelper.Log.Info(string.Format("案件信息:{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10}", new object[] { strArray3[0], strArray3[1], strArray3[2], strArray3[3], strArray3[4], strArray3[5], strArray3[6], strArray3[7], strArray3[8], strArray3[9], strArray3[10] })); this._12345.StartEvents(strArray3[0], strArray3[1], strArray3[2], strArray3[3], strArray3[4], strArray3[5], strArray3[6], strArray3[7], strArray3[8], strArray3[9], strArray3[10]); LogHelper.Log.Info(string.Format("3插入案件成功", new object[0])); } } catch (Exception exception) { LogHelper.Log.Error("插入案件异常：" + exception); } } } LogHelper.Log.Info(ids2.Count == 0 ? DateTime.Now + "现在没有案件更新" : string.Concat(new object[] { DateTime.Now, " 案件更新", ids2.Count, "条" })); } } public void Get12345Data(Guid TargetId, Guid DistrictID, Guid StreetID) { //模拟登陆 string strResult = ReturnHtml(); int totalPageCount = 90; if (totalPageCount > 1) { int testeC = 0; for (int hk = 0; hk < 30; hk++) { if (hk > totalPageCount - 1)//不足10页就提前中断 break; dicList.Clear(); Dictionary<string, string> tempDic = Get12345ProjectCodes(hk);//获取案件的编号列表 List<string> ids = tempDic.Keys.ToList<string>(); List<string> ids2 = new List<string>(); DataTable dt = new DataTable(); dt = _12345.GetEvent_id(); if (dicList != null) { for (int i = 0; i < dicList.Count; i++) { if (!string.IsNullOrEmpty(ids[i])) { if (dt.Select(string.Format("c_event_id='{0}'", ids[i])).Length > 0) { LogHelper.Log.Debug(string.Format("案件ID{0}的案件已存在系统中,第{1},总循环数{2},获取页面数{3}", ids[i], i, dicList.Count,hk)); } else { ids2.Add(ids[i]); LogHelper.Log.Debug(string.Format("案件ID{0}将进入系统,第{1},总循环数{2},获取页面数{3}", ids[i], i, dicList.Count, hk)); } } } int count = 0; LogHelper.Log.Info("每次获取的案件ID集合数量" + ids2.Count); for (int i = 0; i < ids2.Count; i++) { if (ids2[i] != null) { try { count++; string[] nums = Get12345ContentDetail(ids2[i]);//获取小类信息 LogHelper.Log.Debug("1获取内容页数据成功！"); if (nums != null) { testeC++; //***nums0/案件提交用户,nums1/案件举报时间, //nums2/举报电话,nums3/举报电话1, //nums4/举报电话2,nums5/案件举报时间,nums6/案件举报时间, //nums7/12345案件号,nums8/举报电话,nums9/备注 LogHelper.Log.Debug(string.Format(@"案件信息:{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10}；目标环节的办理者Id：{11}", nums[0], nums[1], nums[2] , nums[3], nums[4], nums[5], nums[6], nums[7], nums[8], nums[9], nums[10], TargetId)); _12345.StartEvents_New(nums[0], nums[1], nums[2], nums[3], nums[4], nums[5], nums[6], nums[7], nums[8], nums[9], nums[10], TargetId, DistrictID, StreetID); LogHelper.Log.Info("抓取案件数：" + testeC + "。1插入数据库编号" + nums[7]); } } catch (Exception ex) { LogHelper.Log.Error("插入案件异常：" + ex); } } } if (count == 0)//判断是否有案件跟新 { LogHelper.Log.Info(DateTime.Now + "现在没有案件更新"); } else { LogHelper.Log.Info(DateTime.Now + " 案件更新" + count + "条"); } } } } else { Dictionary<string, string> dicList = this.Get12345ProjectCodes(); List<string> ids = dicList.Keys.ToList<string>(); List<string> ids2 = new List<string>(); DataTable table = new DataTable(); table = this._12345.GetEvent_id(); if (ids != null) { for (int i = 0; i < ids.Count; i++) { if (!string.IsNullOrEmpty(ids[i])) { if (table.Select(string.Format("c_event_id='{0}'", ids[i])).Length > 0) { LogHelper.Log.Debug(string.Format("案件ID{0}的案件已存在系统中", ids[i])); } else { ids2.Add(ids[i]); } } } for (int i = 0; i < ids2.Count; i++) { if (ids2[i] != null) { try { string[] strArray3 = this.Get12345ContentDetail(ids2[i]); LogHelper.Log.Info("获取内容页数据成功！"); if (strArray3 != null) { LogHelper.Log.Info(string.Format("案件信息:{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10}；目标环节的办理者Id：{11}", new object[] { strArray3[0], strArray3[1], strArray3[2], strArray3[3], strArray3[4], strArray3[5], strArray3[6], strArray3[7], strArray3[8], strArray3[9], strArray3[10], TargetId })); this._12345.StartEvents_New(strArray3[0], strArray3[1], strArray3[2], strArray3[3], strArray3[4], strArray3[5], strArray3[6], strArray3[7], strArray3[8], strArray3[9], strArray3[10], TargetId, DistrictID, StreetID); LogHelper.Log.Info(string.Format("2插入案件成功", new object[0])); } } catch (Exception exception) { LogHelper.Log.Error("插入案件异常：" + exception); } } } LogHelper.Log.Info(ids2.Count == 0 ? DateTime.Now + "现在没有案件更新" : string.Concat(new object[] { DateTime.Now, " 案件更新", ids2.Count, "条" })); } } } private void RecursionHtmlNode(INode htmlNode, bool siblingRequired) { if (htmlNode == null) return; try { if (htmlNode is ITag) { ITag tag = (htmlNode as ITag); if (!tag.IsEndTag()) { //if (tag.TagName.ToLower().Trim() == "span") if (tag.TagName.ToLower().Trim() == "b") { if (((Winista.Text.HtmlParser.Nodes.TagNode)(tag)).NextSibling.GetText().Contains("工单编号")) { string key = htmlNode.NextSibling.ToPlainTextString().Split(new char[] { '：' })[1].Trim(); lastKey = key; if (dicList.ContainsKey(key) == false) { dicList.Add(key, ""); } } if (((Winista.Text.HtmlParser.Nodes.TagNode)(tag)).NextSibling.GetText().Contains("所属流程")) { if (dicList.ContainsKey(lastKey)) { dicList[lastKey] = htmlNode.NextSibling.ToPlainTextString().Trim(); } } } } } //获取节点间的内容 if (htmlNode.Children != null && htmlNode.Children.Count > 0) { RecursionHtmlNode(htmlNode.FirstChild, true); } //the sibling nodes if (siblingRequired) { INode sibling = htmlNode.NextSibling; while (sibling != null) { RecursionHtmlNode(sibling, false); sibling = sibling.NextSibling; } } } catch (Exception ex) { LogHelper.Log.Error(string.Format("获取内容页数据异常：{0}", ex)); } } private string ReturnHtmlContent(string oldStr) { int startIndex = -1,endIndex=-1; startIndex = oldStr.IndexOf("<script"); endIndex = oldStr.IndexOf("</script>"); if (startIndex != -1 && endIndex!=-1) { oldStr=oldStr.Remove(startIndex, endIndex+9 -startIndex); oldStr=ReturnHtmlContent(oldStr); } return oldStr; } private Dictionary<string,string> Get12345ProjectCodes() { Exception exception; try { string webPageContent = this.GetWebPageContent("http://113.107.142.14:12345/app/Deal/Pages/replylist.aspx"); if (!string.IsNullOrEmpty(webPageContent)) { try { webPageContent = ReturnHtmlContent(webPageContent); Lexer lexer = new Lexer(webPageContent); Parser parser = new Parser(lexer); NodeList htmlNodes = parser.Parse(null); for (int i = 0; i < htmlNodes.Count; i++) { RecursionHtmlNode(htmlNodes[i], false); } return dicList; } catch (Exception exception1) { exception = exception1; LogHelper.Log.Error(string.Format("获取内容页数据异常：{0}", exception)); } } } catch (Exception exception2) { exception = exception2; LogHelper.Log.Error(string.Format("抓取内容页异常：{0}", exception)); } return null; } Dictionary<string, string> Get12345ProjectCodes(int PageCount) { #region try { string strResult = ReturnHtml(PageCount); if (!string.IsNullOrEmpty(strResult)) { try { if (!string.IsNullOrEmpty(strResult)) { try { strResult = ReturnHtmlContent(strResult); Lexer lexer = new Lexer(strResult); Parser parser = new Parser(lexer); NodeList htmlNodes = parser.Parse(null); for (int i = 0; i < htmlNodes.Count; i++) { RecursionHtmlNode(htmlNodes[i], false); } LogHelper.Log.Error(string.Format("获取的数据量：{0}", dicList.Count)); return dicList; } catch (Exception exception1) { LogHelper.Log.Error(string.Format("获取内容页数据异常：{0}", exception1)); } } LogHelper.Log.Error(string.Format("获取的数据量：{0}", dicList.Count)); return dicList; } catch (Exception ex) { LogHelper.Log.Error(string.Format("抓取内容页异常：{0}", ex)); } } } catch (Exception ex) { LogHelper.Log.Error(string.Format("抓取内容页异常：{0}，当前页：{1}", ex, PageCount)); } return dicList; #endregion } private string GetWebPageContent(string url) { string str2; try { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); request.Timeout = 500000; request.CookieContainer = this.cc; HttpWebResponse response = (HttpWebResponse)request.GetResponse(); using (StreamReader reader = new StreamReader(response.GetResponseStream())) { str2 = reader.ReadToEnd(); } } catch (Exception exception) { LogHelper.Log.Error("请求内容页：" + exception); str2 = string.Empty; } return str2; } private void Login() { this.GetWebPageContent("http://113.107.142.14:12345/app/Deal/Pages/replylist.aspx"); LogHelper.Log.Info("获取列表页数据成功"); } private void Login(string login_name, string pwss_word) { this.GetWebPageContent("http://113.107.142.14:12345/app/Deal/Pages/replylist.aspx"); LogHelper.Log.Info("获取列表页数据成功"); } /// <summary> /// 获取总页数 /// </summary> /// <param name="totalPageCount"></param> void GetTotalPageCount(ref int totalPageCount) { try { string strResult = ReturnHtml(); //获取总案件数量 int testCount = strResult.IndexOf("class='waitpage activied'"); int laseSpanCount = strResult.LastIndexOf("</span>"); //获取每页案件数量 //int testCount = 30; //获取总的页数 totalPageCount = 0; } catch (Exception ex) { LogHelper.Log.Error(string.Format("抓取内容页异常：{0}", ex)); } } private static string ReturnHtml() { Dictionary<string, string> postParams = new Dictionary<string, string>(); postParams.Add("__EVENTTARGET", "btnLogin"); postParams.Add("__EVENTARGUMENT", ""); postParams.Add("__VIEWSTATE", "/wEPDwUKMTQ3ODc3MDg4Nw9kFgICAw9kFgICAw8PZBYCHgV2YWx1ZQUgOUJGMzk5RjUyQTJDN0JFMDMyNURDQUREMDY3RTA4REVkGAEFHl9fQ29udHJvbHNSZXF1aXJlUG9zdEJhY2tLZXlfXxYBBQdjYlRydXN0JK18zzTQsBhXx3/E/THqcjXiDMtvMA5/xVWK6vwjVZk="); postParams.Add("__EVENTVALIDATION", "/wEWBgL7/qTPBwL64ITXCAKT/Zj2BALg0rm0AwLR5qxWAoLch4YMUR4smh/9EOUQ9f0Eg0KWSFu1KOyQeWNxVFQei1Jfb6w="); postParams.Add("txbAccount", "admin_pjcg"); postParams.Add("txbPassword", "pjcg.12345"); postParams.Add("cbTrust", "on"); postParams.Add("rmk", "9BF399F52A2C7BE0325DCADD067E08DE"); //登陆地址 string getViewStateAndEventValidationLoginUrl = "http://113.107.142.14:12345/login.aspx"; //目标地址 string getDataUrl = "http://113.107.142.14:12345/app/Deal/Pages/replylist.aspx"; CookieContainer cookieContainer = new CookieContainer(); /////////////////////////////////////////////////// // 1.打开 MyLogin.aspx 页面，获得 GetVeiwState & EventValidation /////////////////////////////////////////////////// // 设置打开页面的参数 HttpWebRequest request = WebRequest.Create(getViewStateAndEventValidationLoginUrl) as HttpWebRequest; request.Method = "GET"; request.KeepAlive = false; request.AllowAutoRedirect = false; // 接收返回的页面 HttpWebResponse response = request.GetResponse() as HttpWebResponse; System.IO.Stream responseStream = response.GetResponseStream(); System.IO.StreamReader reader = new System.IO.StreamReader(responseStream, Encoding.UTF8); string strResult = reader.ReadToEnd(); /////////////////////////////////////////////////// // 2.自动填充并提交 Login.aspx 页面，提交Login.aspx页面，来保存Cookie /////////////////////////////////////////////////// // 要提交的字符串数据。格式形如:user=uesr1&password=123 string postString = ""; foreach (KeyValuePair<string, string> de in postParams) { //把提交按钮中的中文字符转换成url格式，以防中文或空格等信息 postString += System.Web.HttpUtility.UrlEncode(de.Key.ToString()) + "=" + System.Web.HttpUtility.UrlEncode(de.Value.ToString()) + "&"; } // 将提交的字符串数据转换成字节数组 byte[] postData = Encoding.ASCII.GetBytes(postString); // 设置提交的相关参数 request = WebRequest.Create(getViewStateAndEventValidationLoginUrl) as HttpWebRequest; request.Method = "POST"; request.KeepAlive = false; request.ContentType = "application/x-www-form-urlencoded"; request.CookieContainer = cookieContainer; request.ContentLength = postData.Length; request.AllowAutoRedirect = false; // 提交请求数据 System.IO.Stream outputStream = request.GetRequestStream(); outputStream.Write(postData, 0, postData.Length); outputStream.Close(); // 接收返回的页面 response = request.GetResponse() as HttpWebResponse; responseStream = response.GetResponseStream(); reader = new System.IO.StreamReader(responseStream, Encoding.UTF8); strResult = reader.ReadToEnd(); /////////////////////////////////////////////////// // 3.打开需要抓取数据的页面 /////////////////////////////////////////////////// // 设置打开页面的参数 request = WebRequest.Create(getDataUrl) as HttpWebRequest; request.Method = "GET"; request.KeepAlive = false; request.CookieContainer = cookieContainer; // 接收返回的页面 response = request.GetResponse() as HttpWebResponse; responseStream = response.GetResponseStream(); reader = new System.IO.StreamReader(responseStream, Encoding.UTF8); strResult = reader.ReadToEnd(); /////////////////////////////////////////////////// // 4.分析返回的页面 /////////////////////////////////////////////////// return strResult; } private static string ReturnHtml(string id) { Dictionary<string, string> postParams = new Dictionary<string, string>(); postParams.Add("__EVENTTARGET", "btnLogin"); postParams.Add("__EVENTARGUMENT", ""); postParams.Add("__VIEWSTATE", "/wEPDwUKMTQ3ODc3MDg4Nw9kFgICAw9kFgICAw8PZBYCHgV2YWx1ZQUgOUJGMzk5RjUyQTJDN0JFMDMyNURDQUREMDY3RTA4REVkGAEFHl9fQ29udHJvbHNSZXF1aXJlUG9zdEJhY2tLZXlfXxYBBQdjYlRydXN0JK18zzTQsBhXx3/E/THqcjXiDMtvMA5/xVWK6vwjVZk="); postParams.Add("__EVENTVALIDATION", "/wEWBgL7/qTPBwL64ITXCAKT/Zj2BALg0rm0AwLR5qxWAoLch4YMUR4smh/9EOUQ9f0Eg0KWSFu1KOyQeWNxVFQei1Jfb6w="); postParams.Add("txbAccount", "admin_pjcg"); postParams.Add("txbPassword", "pjcg.12345"); postParams.Add("cbTrust", "on"); postParams.Add("rmk", "9BF399F52A2C7BE0325DCADD067E08DE"); //登陆地址 string getViewStateAndEventValidationLoginUrl = "http://113.107.142.14:12345/login.aspx"; //目标地址 string getDataUrl = string.Format("http://113.107.142.14:12345/app/Deal/Pages/replyadv.aspx?id={0}", id); CookieContainer cookieContainer = new CookieContainer(); /////////////////////////////////////////////////// // 1.打开 MyLogin.aspx 页面，获得 GetVeiwState & EventValidation /////////////////////////////////////////////////// // 设置打开页面的参数 HttpWebRequest request = WebRequest.Create(getViewStateAndEventValidationLoginUrl) as HttpWebRequest; request.Method = "GET"; request.KeepAlive = false; request.AllowAutoRedirect = false; // 接收返回的页面 HttpWebResponse response = request.GetResponse() as HttpWebResponse; System.IO.Stream responseStream = response.GetResponseStream(); System.IO.StreamReader reader = new System.IO.StreamReader(responseStream, Encoding.UTF8); string strResult = reader.ReadToEnd(); /////////////////////////////////////////////////// // 2.自动填充并提交 Login.aspx 页面，提交Login.aspx页面，来保存Cookie /////////////////////////////////////////////////// // 要提交的字符串数据。格式形如:user=uesr1&password=123 string postString = ""; foreach (KeyValuePair<string, string> de in postParams) { //把提交按钮中的中文字符转换成url格式，以防中文或空格等信息 postString += System.Web.HttpUtility.UrlEncode(de.Key.ToString()) + "=" + System.Web.HttpUtility.UrlEncode(de.Value.ToString()) + "&"; } // 将提交的字符串数据转换成字节数组 byte[] postData = Encoding.ASCII.GetBytes(postString); // 设置提交的相关参数 request = WebRequest.Create(getViewStateAndEventValidationLoginUrl) as HttpWebRequest; request.Method = "POST"; request.KeepAlive = false; request.ContentType = "application/x-www-form-urlencoded"; request.CookieContainer = cookieContainer; request.ContentLength = postData.Length; request.AllowAutoRedirect = false; // 提交请求数据 System.IO.Stream outputStream = request.GetRequestStream(); outputStream.Write(postData, 0, postData.Length); outputStream.Close(); // 接收返回的页面 response = request.GetResponse() as HttpWebResponse; responseStream = response.GetResponseStream(); reader = new System.IO.StreamReader(responseStream, Encoding.UTF8); strResult = reader.ReadToEnd(); /////////////////////////////////////////////////// // 3.打开需要抓取数据的页面 /////////////////////////////////////////////////// // 设置打开页面的参数 request = WebRequest.Create(getDataUrl) as HttpWebRequest; request.Method = "GET"; request.KeepAlive = false; request.CookieContainer = cookieContainer; // 接收返回的页面 response = request.GetResponse() as HttpWebResponse; responseStream = response.GetResponseStream(); reader = new System.IO.StreamReader(responseStream, Encoding.UTF8); strResult = reader.ReadToEnd(); /////////////////////////////////////////////////// // 4.分析返回的页面 /////////////////////////////////////////////////// return strResult; } private static string ReturnHtml(int page) { Dictionary<string, string> postParams = new Dictionary<string, string>(); postParams.Add("__EVENTTARGET", "btnLogin"); postParams.Add("__EVENTARGUMENT", ""); postParams.Add("__VIEWSTATE", "/wEPDwUKMTQ3ODc3MDg4Nw9kFgICAw9kFgICAw8PZBYCHgV2YWx1ZQUgOUJGMzk5RjUyQTJDN0JFMDMyNURDQUREMDY3RTA4REVkGAEFHl9fQ29udHJvbHNSZXF1aXJlUG9zdEJhY2tLZXlfXxYBBQdjYlRydXN0JK18zzTQsBhXx3/E/THqcjXiDMtvMA5/xVWK6vwjVZk="); postParams.Add("__EVENTVALIDATION", "/wEWBgL7/qTPBwL64ITXCAKT/Zj2BALg0rm0AwLR5qxWAoLch4YMUR4smh/9EOUQ9f0Eg0KWSFu1KOyQeWNxVFQei1Jfb6w="); postParams.Add("txbAccount", "admin_pjcg"); postParams.Add("txbPassword", "pjcg.12345"); postParams.Add("cbTrust", "on"); postParams.Add("rmk", "9BF399F52A2C7BE0325DCADD067E08DE"); //登陆地址 string getViewStateAndEventValidationLoginUrl = "http://113.107.142.14:12345/login.aspx"; //目标地址 string getDataUrl = string.Format("http://113.107.142.14:12345/app/Deal/Pages/replylist.aspx?page={0}&year={1}", page, DateTime.Now.Year); CookieContainer cookieContainer = new CookieContainer(); /////////////////////////////////////////////////// // 1.打开 MyLogin.aspx 页面，获得 GetVeiwState & EventValidation /////////////////////////////////////////////////// // 设置打开页面的参数 //HttpWebRequest request = WebRequest.Create(getViewStateAndEventValidationLoginUrl) as HttpWebRequest; HttpWebRequest request = WebRequest.Create(getViewStateAndEventValidationLoginUrl) as HttpWebRequest; request.Method = "GET"; request.KeepAlive = false; request.AllowAutoRedirect = false; // 接收返回的页面 HttpWebResponse response = request.GetResponse() as HttpWebResponse; System.IO.Stream responseStream = response.GetResponseStream(); System.IO.StreamReader reader = new System.IO.StreamReader(responseStream, Encoding.UTF8); string strResult = reader.ReadToEnd(); /////////////////////////////////////////////////// // 2.自动填充并提交 Login.aspx 页面，提交Login.aspx页面，来保存Cookie /////////////////////////////////////////////////// // 要提交的字符串数据。格式形如:user=uesr1&password=123 string postString = ""; foreach (KeyValuePair<string, string> de in postParams) { //把提交按钮中的中文字符转换成url格式，以防中文或空格等信息 postString += System.Web.HttpUtility.UrlEncode(de.Key.ToString()) + "=" + System.Web.HttpUtility.UrlEncode(de.Value.ToString()) + "&"; } // 将提交的字符串数据转换成字节数组 byte[] postData = Encoding.ASCII.GetBytes(postString); // 设置提交的相关参数 request = WebRequest.Create(getViewStateAndEventValidationLoginUrl) as HttpWebRequest; request.Method = "POST"; request.KeepAlive = false; request.ContentType = "application/x-www-form-urlencoded"; request.CookieContainer = cookieContainer; request.ContentLength = postData.Length; request.AllowAutoRedirect = false; // 提交请求数据 System.IO.Stream outputStream = request.GetRequestStream(); outputStream.Write(postData, 0, postData.Length); outputStream.Close(); // 接收返回的页面 response = request.GetResponse() as HttpWebResponse; responseStream = response.GetResponseStream(); reader = new System.IO.StreamReader(responseStream, Encoding.UTF8); strResult = reader.ReadToEnd(); /////////////////////////////////////////////////// // 3.打开需要抓取数据的页面 /////////////////////////////////////////////////// // 设置打开页面的参数 request = WebRequest.Create(getDataUrl) as HttpWebRequest; request.Method = "GET"; request.KeepAlive = false; request.CookieContainer = cookieContainer; // 接收返回的页面 response = request.GetResponse() as HttpWebResponse; responseStream = response.GetResponseStream(); reader = new System.IO.StreamReader(responseStream, Encoding.UTF8); strResult = reader.ReadToEnd(); /////////////////////////////////////////////////// // 4.分析返回的页面 /////////////////////////////////////////////////// return strResult; } } }
标签：
版权申明：本站文章部分自网络，如有侵权，请联系：west999com@outlook.com
特别注意：本站所有转载文章言论不代表本站观点，本站所提供的摄影照片，插画，设计作品，如需使用，请与原作者联系，版权归原作者所有
相关文章
- asp.net jQuery Ajax用户登录功能的实现 2020-03-15
- .NET微信开发之PC 端微信扫码注册和登录功能实现 2020-02-23
- asp.net网站实现接入QQ登录示例代码 2020-02-17
- ASP.NET Core集成微信登录 2020-02-17
- Asp.net mvc验证用户登录之Forms实现详解 2020-02-05
IDC资讯： 主机资讯 注册资讯 托管资讯 vps资讯 网站建设
网站运营： 建站经验 策划盈利 搜索优化 网站推广 免费资源
网络编程： Asp.Net编程 Asp编程 Php编程 Xml编程 Access Mssql Mysql 其它
服务器技术： Web服务器 Ftp服务器 Mail服务器 Dns服务器 安全防护
软件技巧： 其它软件 Word Excel Powerpoint Ghost Vista QQ空间 QQ FlashGet 迅雷
网页制作： FrontPages Dreamweaver Javascript css photoshop fireworks Flash
热门词条
最新资讯
热门关注
热门标签