fork download
  1. using Newtonsoft.Json;
  2. using System;
  3. using System.Collections.Generic;
  4. using System.IO;
  5. using System.Linq;
  6. using System.Net;
  7. using System.Text;
  8. using System.Threading;
  9. using System.Threading.Tasks;
  10.  
  11. namespace GalleryExplorer.Core
  12. {
  13.  
  14. public class InstanceMonitor
  15. {
  16. public static Dictionary<string, object> Instances = new Dictionary<string, object>();
  17. }
  18.  
  19. /// <summary>
  20. /// Lazy구현을 쉽게 해주는 클래스입니다.
  21. /// </summary>
  22. /// <typeparam name="T"></typeparam>
  23. public class ILazy<T>
  24. where T : new()
  25. {
  26. private static readonly Lazy<T> instance = new Lazy<T>(() =>
  27. {
  28. T instance = new T();
  29. InstanceMonitor.Instances.Add(instance.GetType().Name.ToLower(), instance);
  30. return instance;
  31. });
  32. public static T Instance => instance.Value;
  33. public static bool IsValueCreated => instance.IsValueCreated;
  34. }
  35.  
  36. public class ProxyList : ILazy<ProxyList>
  37. {
  38. public const string Name = "proxy.txt";
  39.  
  40. public List<string> List { get; set; } = new List<string>();
  41.  
  42. public ProxyList()
  43. {
  44. var full_path = Path.Combine(Directory.GetCurrentDirectory(), Name);
  45. if (!File.Exists(full_path))
  46. {
  47. Logger.Instance.PushError("[Proxy] 'proxy.txt' not found!");
  48. return;
  49. }
  50. var txt = File.ReadAllLines(full_path);
  51. foreach (var line in txt)
  52. {
  53. if (string.IsNullOrEmpty(line))
  54. break;
  55. List.Add(line.Split(' ')[0].Trim());
  56. }
  57. }
  58.  
  59. public string RandomPick()
  60. => List[new Random().Next(List.Count)];
  61. }
  62.  
  63. public class SettingModel
  64. {
  65. public class NetworkSetting
  66. {
  67. public bool TimeoutInfinite;
  68. public int TimeoutMillisecond;
  69. public int DownloadBufferSize;
  70. public int RetryCount;
  71. public string Proxy;
  72. public bool UsingProxyList;
  73. }
  74.  
  75. public NetworkSetting NetworkSettings;
  76.  
  77. /// <summary>
  78. /// Scheduler Thread Count
  79. /// </summary>
  80. public int ThreadCount;
  81.  
  82. /// <summary>
  83. /// Postprocessor Scheduler Thread Count
  84. /// </summary>
  85. public int PostprocessorThreadCount;
  86.  
  87. /// <summary>
  88. /// Provider Language
  89. /// </summary>
  90. public string Language;
  91.  
  92. /// <summary>
  93. /// Parent Path for Downloading
  94. /// </summary>
  95. public string SuperPath;
  96.  
  97. /// <summary>
  98. /// WebSocket Server Address
  99. /// </summary>
  100. public string ConnectionAddress;
  101. }
  102.  
  103. public class Settings : ILazy<Settings>
  104. {
  105. public SettingModel Model { get; set; }
  106. public SettingModel.NetworkSetting Network { get { return Model.NetworkSettings; } }
  107.  
  108. public Settings()
  109. {
  110. Model = new SettingModel
  111. {
  112. Language = GetLanguageKey(),
  113. ThreadCount = Environment.ProcessorCount,
  114. //ThreadCount = 3,
  115. PostprocessorThreadCount = 3,
  116.  
  117. NetworkSettings = new SettingModel.NetworkSetting
  118. {
  119. TimeoutInfinite = false,
  120. TimeoutMillisecond = 10000,
  121. DownloadBufferSize = 131072,
  122. RetryCount = 10,
  123. UsingProxyList = false,
  124. },
  125.  
  126. };
  127. }
  128.  
  129. public static string GetLanguageKey()
  130. {
  131. var lang = Thread.CurrentThread.CurrentCulture.ToString();
  132. var language = "all";
  133. switch (lang)
  134. {
  135. case "ko-KR":
  136. language = "korean";
  137. break;
  138.  
  139. case "ja-JP":
  140. language = "japanese";
  141. break;
  142.  
  143. case "en-US":
  144. language = "english";
  145. break;
  146. }
  147. return language;
  148. }
  149. }
  150.  
  151.  
  152. public enum NetPriorityType
  153. {
  154. // ex) Download and save file, large data
  155. Low = 0,
  156. // ex) Download metadata, html file ...
  157. Trivial = 1,
  158. // Pause all processing and force download
  159. Emergency = 2,
  160. }
  161.  
  162. public class NetPriority : IComparable<NetPriority>
  163. {
  164. [JsonProperty]
  165. public NetPriorityType Type { get; set; }
  166. [JsonProperty]
  167. public int TaskPriority { get; set; }
  168.  
  169. public int CompareTo(NetPriority pp)
  170. {
  171. if (Type > pp.Type) return 1;
  172. else if (Type < pp.Type) return -1;
  173.  
  174. return pp.TaskPriority.CompareTo(TaskPriority);
  175. }
  176. }
  177.  
  178. /// <summary>
  179. /// Information of what download for
  180. /// </summary>
  181. [JsonObject(MemberSerialization.OptIn)]
  182. public class NetTask : ISchedulerContents<NetTask, NetPriority>
  183. {
  184. public static NetTask MakeDefault(string url, string cookie = "")
  185. => new NetTask
  186. {
  187. Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
  188. UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
  189. TimeoutInfinite = Settings.Instance.Network.TimeoutInfinite,
  190. TimeoutMillisecond = Settings.Instance.Network.TimeoutMillisecond,
  191. AutoRedirection = true,
  192. RetryWhenFail = true,
  193. RetryCount = Settings.Instance.Network.RetryCount,
  194. DownloadBufferSize = Settings.Instance.Network.DownloadBufferSize,
  195. Priority = new NetPriority() { Type = NetPriorityType.Trivial },
  196. Proxy = !string.IsNullOrEmpty(Settings.Instance.Network.Proxy) ? new WebProxy(Settings.Instance.Network.Proxy) :
  197. Settings.Instance.Network.UsingProxyList ? new WebProxy(ProxyList.Instance.RandomPick()) : null,
  198. Cookie = cookie,
  199. Url = url
  200. };
  201.  
  202. public static NetTask MakeDefaultMobile(string url, string cookie = "")
  203. => new NetTask
  204. {
  205. Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
  206. UserAgent = "Mozilla/5.0 (Android 7.0; Mobile; rv:54.0) Gecko/54.0 Firefox/54.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/603.2.4",
  207. TimeoutInfinite = Settings.Instance.Network.TimeoutInfinite,
  208. TimeoutMillisecond = Settings.Instance.Network.TimeoutMillisecond,
  209. AutoRedirection = true,
  210. RetryWhenFail = true,
  211. RetryCount = Settings.Instance.Network.RetryCount,
  212. DownloadBufferSize = Settings.Instance.Network.DownloadBufferSize,
  213. Priority = new NetPriority() { Type = NetPriorityType.Trivial },
  214. Proxy = !string.IsNullOrEmpty(Settings.Instance.Network.Proxy) ? new WebProxy(Settings.Instance.Network.Proxy) :
  215. Settings.Instance.Network.UsingProxyList ? new WebProxy(ProxyList.Instance.RandomPick()) : null,
  216. Cookie = cookie,
  217. Url = url
  218. };
  219.  
  220. public enum NetError
  221. {
  222. Unhandled = 0,
  223. CannotContinueByCriticalError,
  224. UnknowError, // Check DPI Blocker
  225. UriFormatError,
  226. Aborted,
  227. ManyRetry,
  228. }
  229.  
  230. /* Task Information */
  231.  
  232. [JsonProperty]
  233. public int Index { get; set; }
  234.  
  235. /* Http Information */
  236.  
  237. [JsonProperty]
  238. public string Url { get; set; }
  239. [JsonProperty]
  240. public List<string> FailUrls { get; set; }
  241. [JsonProperty]
  242. public string Accept { get; set; }
  243. [JsonProperty]
  244. public string Referer { get; set; }
  245. [JsonProperty]
  246. public string UserAgent { get; set; }
  247. [JsonProperty]
  248. public string Cookie { get; set; }
  249. [JsonProperty]
  250. public Dictionary<string, string> Headers { get; set; }
  251. [JsonProperty]
  252. public Dictionary<string, string> Query { get; set; }
  253. [JsonProperty]
  254. public IWebProxy Proxy { get; set; }
  255.  
  256. /* Detail Information */
  257.  
  258. /// <summary>
  259. /// Text Encoding Information
  260. /// </summary>
  261. public Encoding Encoding { get; set; }
  262.  
  263. /// <summary>
  264. /// Set if you want to download and save file to your own device.
  265. /// </summary>
  266. [JsonProperty]
  267. public bool SaveFile { get; set; }
  268. [JsonProperty]
  269. public string Filename { get; set; }
  270.  
  271. /// <summary>
  272. /// Set if needing only string datas.
  273. /// </summary>
  274. [JsonProperty]
  275. public bool DownloadString { get; set; }
  276.  
  277. /// <summary>
  278. /// Download data to temporary directory on your device.
  279. /// </summary>
  280. [JsonProperty]
  281. public bool DriveCache { get; set; }
  282.  
  283. /// <summary>
  284. /// Download data to memory.
  285. /// </summary>
  286. [JsonProperty]
  287. public bool MemoryCache { get; set; }
  288.  
  289. /// <summary>
  290. /// Retry download when fail to download.
  291. /// </summary>
  292. [JsonProperty]
  293. public bool RetryWhenFail { get; set; }
  294. [JsonProperty]
  295. public int RetryCount { get; set; }
  296.  
  297. /// <summary>
  298. /// Timeout settings
  299. /// </summary>
  300. [JsonProperty]
  301. public bool TimeoutInfinite { get; set; }
  302. [JsonProperty]
  303. public int TimeoutMillisecond { get; set; }
  304.  
  305. [JsonProperty]
  306. public int DownloadBufferSize { get; set; }
  307.  
  308. [JsonProperty]
  309. public bool AutoRedirection { get; set; }
  310.  
  311. [JsonProperty]
  312. public bool NotifyOnlySize { get; set; }
  313.  
  314. [JsonProperty]
  315. public bool GetStream { get; set; }
  316.  
  317. /* Callback Functions */
  318.  
  319. public Action<long> SizeCallback;
  320. public Action<long> DownloadCallback;
  321. public Action StartCallback;
  322. public Action CompleteCallback;
  323. public Action<string> CompleteCallbackString;
  324. public Action<byte[]> CompleteCallbackBytes;
  325. public Action<Stream> CompleteCallbackStream;
  326. public Action<CookieCollection> CookieReceive;
  327. public Action<string> HeaderReceive;
  328. public Action CancleCallback;
  329.  
  330. /// <summary>
  331. /// Return total downloaded size
  332. /// </summary>
  333. public Action<int> RetryCallback;
  334. public Action<NetError> ErrorCallback;
  335.  
  336. /* For NetField */
  337.  
  338. public bool Aborted;
  339. public HttpWebRequest Request;
  340. public CancellationToken Cancel;
  341.  
  342. /* Post Processor */
  343.  
  344. public Action StartPostprocessorCallback;
  345. }
  346.  
  347.  
  348. public class UpdatableHeapElements<T> : IComparable<T>
  349. where T : IComparable<T>
  350. {
  351. public T data;
  352. public int index;
  353. public static UpdatableHeapElements<T> Create(T data, int index)
  354. => new UpdatableHeapElements<T> { data = data, index = index };
  355. public int CompareTo(T obj)
  356. => data.CompareTo(obj);
  357. }
  358.  
  359. public class UpdatableHeap<S, T, C>
  360. where S : IComparable<S>
  361. where T : UpdatableHeapElements<S>, IComparable<S>
  362. where C : IComparer<S>, new()
  363. {
  364. List<T> heap;
  365. C comp;
  366.  
  367. public UpdatableHeap(int capacity = 256)
  368. {
  369. heap = new List<T>(capacity);
  370. comp = new C();
  371. }
  372.  
  373. public T Push(S d)
  374. {
  375. var dd = (T)UpdatableHeapElements<S>.Create(d, heap.Count - 1);
  376. heap.Add(dd);
  377. top_down(heap.Count - 1);
  378. return dd;
  379. }
  380.  
  381. public void Pop()
  382. {
  383. heap[0] = heap[heap.Count - 1];
  384. heap[0].index = 0;
  385. heap.RemoveAt(heap.Count - 1);
  386. bottom_up();
  387. }
  388.  
  389. public void Update(T d)
  390. {
  391. int p = (d.index - 1) >> 1;
  392. if (p == d.index)
  393. bottom_up();
  394. else
  395. {
  396. if (comp.Compare(heap[p].data, heap[d.index].data) > 0)
  397. top_down(d.index);
  398. else
  399. bottom_up(d.index);
  400. }
  401. }
  402.  
  403. public S Front => heap[0].data;
  404.  
  405. public int Count { get { return heap.Count; } }
  406.  
  407. private void bottom_up(int x = 0)
  408. {
  409. int l = heap.Count - 1;
  410. while (x < l)
  411. {
  412. int c1 = x * 2 + 1;
  413. int c2 = c1 + 1;
  414.  
  415. //
  416. // x
  417. // / \
  418.   // / \
  419.   // c1 c2
  420. //
  421.  
  422. int c = c1;
  423. if (c2 < l && comp.Compare(heap[c2].data, heap[c1].data) > 0)
  424. c = c2;
  425.  
  426. if (c < l && comp.Compare(heap[c].data, heap[x].data) > 0)
  427. {
  428. swap(c, x);
  429. x = c;
  430. }
  431. else
  432. {
  433. break;
  434. }
  435. }
  436. }
  437.  
  438. private void top_down(int x)
  439. {
  440. while (x > 0)
  441. {
  442. int p = (x - 1) >> 1;
  443. if (comp.Compare(heap[x].data, heap[p].data) > 0)
  444. {
  445. swap(p, x);
  446. x = p;
  447. }
  448. else
  449. break;
  450. }
  451. }
  452.  
  453. private void swap(int i, int j)
  454. {
  455. T t = heap[i];
  456. heap[i] = heap[j];
  457. heap[j] = t;
  458.  
  459. int tt = heap[i].index;
  460. heap[i].index = heap[j].index;
  461. heap[j].index = tt;
  462. }
  463. }
  464.  
  465. public class DefaultHeapComparer<T> : Comparer<T> where T : IComparable<T>
  466. {
  467. public override int Compare(T x, T y)
  468. => x.CompareTo(y);
  469. }
  470.  
  471. public class UpdatableHeap<T> : UpdatableHeap<T, UpdatableHeapElements<T>, DefaultHeapComparer<T>> where T : IComparable<T> { }
  472.  
  473. public interface IScheduler<T>
  474. where T : IComparable<T>
  475. {
  476. void update(UpdatableHeapElements<T> elem);
  477. }
  478.  
  479. public class ISchedulerContents<T, P>
  480. : IComparable<ISchedulerContents<T, P>>
  481. where T : IComparable<T>
  482. where P : IComparable<P>
  483. {
  484. /* Scheduler Information */
  485. P priority;
  486.  
  487. [JsonProperty]
  488. public P Priority { get { return priority; } set { priority = value; if (scheduler != null) scheduler.update(heap_elements); } }
  489. public int CompareTo(ISchedulerContents<T, P> other)
  490. => Priority.CompareTo(other.Priority);
  491.  
  492. public UpdatableHeapElements<T> heap_elements;
  493. public IScheduler<T> scheduler;
  494. }
  495.  
  496. public abstract class IField<T, P>
  497. where T : ISchedulerContents<T, P>
  498. where P : IComparable<P>
  499. {
  500. public abstract void Main(T content);
  501. public ManualResetEvent interrupt = new ManualResetEvent(true);
  502. }
  503.  
  504. /// <summary>
  505. /// Scheduler Interface
  506. /// </summary>
  507. /// <typeparam name="T">Task type</typeparam>
  508. /// <typeparam name="P">Priority type</typeparam>
  509. /// <typeparam name="F">Field type</typeparam>
  510. public class Scheduler<T, P, F>
  511. : IScheduler<T>
  512. where T : ISchedulerContents<T, P>
  513. where P : IComparable<P>
  514. where F : IField<T, P>, new()
  515. {
  516. public UpdatableHeap<T> queue = new UpdatableHeap<T>();
  517.  
  518. public void update(UpdatableHeapElements<T> elem)
  519. {
  520. queue.Update(elem);
  521. }
  522.  
  523. public int thread_count = 0;
  524. public int busy_thread = 0;
  525. public int capacity = 0;
  526.  
  527. public P LatestPriority;
  528.  
  529. public List<Thread> threads = new List<Thread>();
  530. public List<ManualResetEvent> interrupt = new List<ManualResetEvent>();
  531. public List<F> field = new List<F>();
  532.  
  533. object notify_lock = new object();
  534.  
  535. public Scheduler(int capacity = 0, bool use_emergency_thread = false)
  536. {
  537. this.capacity = capacity;
  538.  
  539. if (this.capacity == 0)
  540. this.capacity = Environment.ProcessorCount;
  541.  
  542. thread_count = this.capacity;
  543.  
  544. if (use_emergency_thread)
  545. thread_count += 1;
  546.  
  547. for (int i = 0; i < this.capacity; i++)
  548. {
  549. interrupt.Add(new ManualResetEvent(false));
  550. threads.Add(new Thread(new ParameterizedThreadStart(remote_thread_handler)));
  551. threads.Last().Start(i);
  552. }
  553.  
  554. for (int i = 0; i < this.capacity; i++)
  555. {
  556. field.Add(new F());
  557. }
  558. }
  559.  
  560. private void remote_thread_handler(object i)
  561. {
  562. int index = (int)i;
  563.  
  564. while (true)
  565. {
  566. interrupt[index].WaitOne();
  567.  
  568. T task;
  569.  
  570. lock (queue)
  571. {
  572. if (queue.Count > 0)
  573. {
  574. task = queue.Front;
  575. queue.Pop();
  576. }
  577. else
  578. {
  579. interrupt[index].Reset();
  580. continue;
  581. }
  582. }
  583.  
  584. Interlocked.Increment(ref busy_thread);
  585.  
  586. LatestPriority = task.Priority;
  587.  
  588. field[index].Main(task);
  589.  
  590. Interlocked.Decrement(ref busy_thread);
  591. }
  592. }
  593.  
  594. public void Pause()
  595. {
  596. field.ForEach(x => x.interrupt.Reset());
  597. }
  598.  
  599. public void Resume()
  600. {
  601. field.ForEach(x => x.interrupt.Set());
  602. }
  603.  
  604. public void Notify()
  605. {
  606. interrupt.ForEach(x => x.Set());
  607. }
  608.  
  609. public UpdatableHeapElements<T> Add(T task)
  610. {
  611. task.scheduler = this;
  612. UpdatableHeapElements<T> e;
  613. lock (queue) e = queue.Push(task);
  614. lock (notify_lock) Notify();
  615. return e;
  616. }
  617. }
  618.  
  619. /// <summary>
  620. /// Implementaion of real download procedure
  621. /// </summary>
  622. public class NetField : IField<NetTask, NetPriority>
  623. {
  624. public override void Main(NetTask content)
  625. {
  626. var retry_count = 0;
  627.  
  628. RETRY_PROCEDURE:
  629.  
  630. interrupt.WaitOne();
  631. if (content.Cancel != null && content.Cancel.IsCancellationRequested)
  632. {
  633. content.CancleCallback();
  634. return;
  635. }
  636.  
  637. interrupt.WaitOne();
  638.  
  639. if (content.DownloadString)
  640. Logger.Instance.Push("[NetField] Start download string... " + content.Url);
  641. else if (content.MemoryCache)
  642. Logger.Instance.Push("[NetField] Start download to memory... " + content.Url);
  643. else if (content.SaveFile)
  644. Logger.Instance.Push("[NetField] Start download file... " + content.Url + " to " + content.Filename);
  645.  
  646. REDIRECTION:
  647.  
  648. interrupt.WaitOne();
  649. if (content.Cancel != null && content.Cancel.IsCancellationRequested)
  650. {
  651. content.CancleCallback();
  652. return;
  653. }
  654.  
  655. content.StartCallback?.Invoke();
  656.  
  657. try
  658. {
  659. //
  660. // Initialize http-web-request
  661. //
  662.  
  663. var request = (HttpWebRequest)WebRequest.Create(content.Url);
  664. content.Request = request;
  665.  
  666. request.Accept = content.Accept;
  667. request.UserAgent = content.UserAgent;
  668.  
  669. if (content.Referer != null)
  670. request.Referer = content.Referer;
  671. else
  672. request.Referer = (content.Url.StartsWith("https://") ? "https://" : (content.Url.Split(':')[0] + "//")) + request.RequestUri.Host;
  673.  
  674. if (content.Cookie != null)
  675. request.Headers.Add(HttpRequestHeader.Cookie, content.Cookie);
  676.  
  677. if (content.Headers != null)
  678. content.Headers.ToList().ForEach(p => request.Headers.Add(p.Key, p.Value));
  679.  
  680. if (content.Proxy != null)
  681. request.Proxy = content.Proxy;
  682.  
  683. if (content.TimeoutInfinite)
  684. request.Timeout = Timeout.Infinite;
  685. else
  686. request.Timeout = content.TimeoutMillisecond;
  687.  
  688. request.AllowAutoRedirect = content.AutoRedirection;
  689.  
  690. //
  691. // POST Data
  692. //
  693.  
  694. if (content.Query != null)
  695. {
  696. request.Method = "POST";
  697. request.ContentType = "application/x-www-form-urlencoded";
  698.  
  699. var request_stream = new StreamWriter(request.GetRequestStream());
  700. var query = string.Join("&", content.Query.ToList().Select(x => $"{x.Key}={x.Value}"));
  701. request_stream.Write(query);
  702. request_stream.Close();
  703.  
  704. interrupt.WaitOne();
  705. if (content.Cancel != null && content.Cancel.IsCancellationRequested)
  706. {
  707. content.CancleCallback();
  708. return;
  709. }
  710. }
  711.  
  712. //
  713. // Wait request
  714. //
  715.  
  716. using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
  717. {
  718. if (response.StatusCode == HttpStatusCode.NotFound ||
  719. response.StatusCode == HttpStatusCode.Forbidden ||
  720. response.StatusCode == HttpStatusCode.Unauthorized ||
  721. response.StatusCode == HttpStatusCode.BadRequest ||
  722. response.StatusCode == HttpStatusCode.InternalServerError)
  723. {
  724. //
  725. // Cannot continue
  726. //
  727.  
  728. content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError);
  729. return;
  730. }
  731. else if (response.StatusCode == HttpStatusCode.Moved ||
  732. response.StatusCode == HttpStatusCode.Redirect)
  733. {
  734. if (content.AutoRedirection)
  735. {
  736. var old = content.Url;
  737. content.Url = response.Headers.Get("Location");
  738. Logger.Instance.Push("[NetField] Redirection " + old + " to " + content.Url);
  739. goto REDIRECTION;
  740. }
  741. }
  742. else if (response.StatusCode == HttpStatusCode.OK)
  743. {
  744. interrupt.WaitOne();
  745. if (content.Cancel != null && content.Cancel.IsCancellationRequested)
  746. {
  747. content.CancleCallback();
  748. return;
  749. }
  750.  
  751. content.HeaderReceive?.Invoke(response.Headers.ToString());
  752. content.CookieReceive?.Invoke(response.Cookies);
  753.  
  754. Stream istream = response.GetResponseStream();
  755. Stream ostream = null;
  756.  
  757. if (content.GetStream)
  758. {
  759. content.CompleteCallbackStream?.Invoke(istream);
  760. return;
  761. }
  762.  
  763. if (content.DownloadString || content.MemoryCache)
  764. {
  765. ostream = new MemoryStream();
  766. }
  767. else if (content.DriveCache)
  768. {
  769. // TODO:
  770. }
  771. else
  772. {
  773. ostream = File.OpenWrite(content.Filename);
  774. }
  775.  
  776. content.SizeCallback?.Invoke(response.ContentLength);
  777.  
  778. if (content.NotifyOnlySize)
  779. {
  780. ostream.Close();
  781. istream.Close();
  782. return;
  783. }
  784.  
  785. interrupt.WaitOne();
  786. if (content.Cancel != null && content.Cancel.IsCancellationRequested)
  787. {
  788. content.CancleCallback();
  789. return;
  790. }
  791.  
  792. byte[] buffer = new byte[content.DownloadBufferSize];
  793. long byte_read = 0;
  794.  
  795. //
  796. // Download loop
  797. //
  798.  
  799. do
  800. {
  801. interrupt.WaitOne();
  802. if (content.Cancel != null && content.Cancel.IsCancellationRequested)
  803. {
  804. content.CancleCallback();
  805. return;
  806. }
  807.  
  808. byte_read = istream.Read(buffer, 0, buffer.Length);
  809. ostream.Write(buffer, 0, (int)byte_read);
  810.  
  811. interrupt.WaitOne();
  812. if (content.Cancel != null && content.Cancel.IsCancellationRequested)
  813. {
  814. content.CancleCallback();
  815. return;
  816. }
  817.  
  818. content.DownloadCallback?.Invoke(byte_read);
  819.  
  820. } while (byte_read != 0);
  821.  
  822. //
  823. // Notify Complete
  824. //
  825.  
  826. if (content.DownloadString)
  827. {
  828. if (content.Encoding == null)
  829. content.CompleteCallbackString(Encoding.UTF8.GetString(((MemoryStream)ostream).ToArray()));
  830. else
  831. content.CompleteCallbackString(content.Encoding.GetString(((MemoryStream)ostream).ToArray()));
  832. }
  833. else if (content.MemoryCache)
  834. {
  835. content.CompleteCallbackBytes(((MemoryStream)ostream).ToArray());
  836. }
  837. else
  838. {
  839. content.CompleteCallback?.Invoke();
  840. }
  841.  
  842. ostream.Close();
  843. istream.Close();
  844.  
  845. return;
  846. }
  847. }
  848. }
  849. catch (WebException e)
  850. {
  851. var response = (HttpWebResponse)e.Response;
  852.  
  853. if (response != null && response.StatusCode == HttpStatusCode.Moved)
  854. {
  855. if (content.AutoRedirection)
  856. {
  857. var old = content.Url;
  858. content.Url = response.Headers.Get("Location");
  859. Logger.Instance.Push("[NetField] Redirection " + old + " to " + content.Url);
  860. goto REDIRECTION;
  861. }
  862. }
  863.  
  864. lock (Logger.Instance)
  865. {
  866. Logger.Instance.PushError("[NetField] Web Excpetion - " + e.Message + "\r\n" + e.StackTrace);
  867. Logger.Instance.PushError(content);
  868. }
  869.  
  870. if (content.FailUrls != null && retry_count < content.FailUrls.Count)
  871. {
  872. content.Url = content.FailUrls[retry_count++];
  873. content.RetryCallback?.Invoke(retry_count);
  874.  
  875. lock (Logger.Instance)
  876. {
  877. Logger.Instance.Push($"[NetField] Retry [{retry_count}/{content.RetryCount}]");
  878. Logger.Instance.Push(content);
  879. }
  880. goto RETRY_PROCEDURE;
  881. }
  882.  
  883. if ((response != null && (
  884. response.StatusCode == HttpStatusCode.NotFound ||
  885. response.StatusCode == HttpStatusCode.Forbidden ||
  886. response.StatusCode == HttpStatusCode.Unauthorized ||
  887. response.StatusCode == HttpStatusCode.BadRequest ||
  888. response.StatusCode == HttpStatusCode.InternalServerError)) ||
  889. e.Status == WebExceptionStatus.NameResolutionFailure ||
  890. e.Status == WebExceptionStatus.UnknownError)
  891. {
  892. if (response != null && response.StatusCode == HttpStatusCode.Forbidden && response.Cookies != null)
  893. {
  894. content.CookieReceive?.Invoke(response.Cookies);
  895. return;
  896. }
  897.  
  898. //
  899. // Cannot continue
  900. //
  901.  
  902. if (e.Status == WebExceptionStatus.UnknownError)
  903. {
  904. lock (Logger.Instance)
  905. {
  906. Logger.Instance.PushError("[NetField] Check your Firewall, Router or DPI settings.");
  907. Logger.Instance.PushError("[NetField] If you continue to receive this error, please contact developer.");
  908. }
  909.  
  910. content.ErrorCallback?.Invoke(NetTask.NetError.UnknowError);
  911. }
  912. else
  913. {
  914. content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError);
  915. }
  916.  
  917. return;
  918. }
  919. }
  920. catch (UriFormatException e)
  921. {
  922. lock (Logger.Instance)
  923. {
  924. Logger.Instance.PushError("[NetField] URI Exception - " + e.Message + "\r\n" + e.StackTrace);
  925. Logger.Instance.PushError(content);
  926. }
  927.  
  928. //
  929. // Cannot continue
  930. //
  931.  
  932. content.ErrorCallback?.Invoke(NetTask.NetError.UriFormatError);
  933. return;
  934. }
  935. catch (Exception e)
  936. {
  937. lock (Logger.Instance)
  938. {
  939. Logger.Instance.PushError("[NetField] Unhandled Excpetion - " + e.Message + "\r\n" + e.StackTrace);
  940. Logger.Instance.PushError(content);
  941. }
  942. }
  943.  
  944. //
  945. // Request Aborted
  946. //
  947.  
  948. if (content.Aborted)
  949. {
  950. content.ErrorCallback?.Invoke(NetTask.NetError.Aborted);
  951. return;
  952. }
  953.  
  954. //
  955. // Retry
  956. //
  957.  
  958. if (content.FailUrls != null && retry_count < content.FailUrls.Count)
  959. {
  960. content.Url = content.FailUrls[retry_count++];
  961. content.RetryCallback?.Invoke(retry_count);
  962.  
  963. lock (Logger.Instance)
  964. {
  965. Logger.Instance.Push($"[NetField] Retry [{retry_count}/{content.RetryCount}]");
  966. Logger.Instance.Push(content);
  967. }
  968. goto RETRY_PROCEDURE;
  969. }
  970.  
  971. if (content.RetryWhenFail)
  972. {
  973. if (content.RetryCount > retry_count)
  974. {
  975. retry_count += 1;
  976.  
  977. content.RetryCallback?.Invoke(retry_count);
  978.  
  979. lock (Logger.Instance)
  980. {
  981. Logger.Instance.Push($"[NetField] Retry [{retry_count}/{content.RetryCount}]");
  982. Logger.Instance.Push(content);
  983. }
  984. goto RETRY_PROCEDURE;
  985. }
  986.  
  987. //
  988. // Many retry
  989. //
  990.  
  991. lock (Logger.Instance)
  992. {
  993. Logger.Instance.Push($"[NetField] Many Retry");
  994. Logger.Instance.Push(content);
  995. }
  996. content.ErrorCallback?.Invoke(NetTask.NetError.ManyRetry);
  997. }
  998.  
  999. content.ErrorCallback?.Invoke(NetTask.NetError.Unhandled);
  1000. }
  1001. }
  1002.  
  1003. public class NetScheduler : Scheduler<NetTask, NetPriority, NetField>
  1004. {
  1005. public NetScheduler(int capacity = 0, bool use_emergency_thread = false)
  1006. : base(capacity, use_emergency_thread) { }
  1007. }
  1008.  
  1009. public class NetTools
  1010. {
  1011. public static NetScheduler Scheduler = new NetScheduler(Settings.Instance.Model.ThreadCount);
  1012.  
  1013. public static List<string> DownloadStrings(List<string> urls, string cookie = "", Action complete = null)
  1014. {
  1015. var interrupt = new ManualResetEvent(false);
  1016. var result = new string[urls.Count];
  1017. var count = urls.Count;
  1018. int iter = 0;
  1019.  
  1020. foreach (var url in urls)
  1021. {
  1022. var itertmp = iter;
  1023. var task = NetTask.MakeDefault(url);
  1024. task.DownloadString = true;
  1025. task.CompleteCallbackString = (str) =>
  1026. {
  1027. result[itertmp] = str;
  1028. if (Interlocked.Decrement(ref count) == 0)
  1029. interrupt.Set();
  1030. complete?.Invoke();
  1031. };
  1032. task.ErrorCallback = (code) =>
  1033. {
  1034. if (Interlocked.Decrement(ref count) == 0)
  1035. interrupt.Set();
  1036. };
  1037. task.Cookie = cookie;
  1038. Scheduler.Add(task);
  1039. iter++;
  1040. }
  1041.  
  1042. interrupt.WaitOne();
  1043.  
  1044. return result.ToList();
  1045. }
  1046.  
  1047. public static List<string> DownloadStrings(List<NetTask> tasks, string cookie = "", Action complete = null)
  1048. {
  1049. var interrupt = new ManualResetEvent(false);
  1050. var result = new string[tasks.Count];
  1051. var count = tasks.Count;
  1052. int iter = 0;
  1053.  
  1054. foreach (var task in tasks)
  1055. {
  1056. var itertmp = iter;
  1057. task.DownloadString = true;
  1058. task.CompleteCallbackString = (str) =>
  1059. {
  1060. result[itertmp] = str;
  1061. if (Interlocked.Decrement(ref count) == 0)
  1062. interrupt.Set();
  1063. complete?.Invoke();
  1064. };
  1065. task.ErrorCallback = (code) =>
  1066. {
  1067. if (Interlocked.Decrement(ref count) == 0)
  1068. interrupt.Set();
  1069. };
  1070. task.Cookie = cookie;
  1071. Scheduler.Add(task);
  1072. iter++;
  1073. }
  1074.  
  1075. interrupt.WaitOne();
  1076.  
  1077. return result.ToList();
  1078. }
  1079.  
  1080. public static string DownloadString(string url)
  1081. {
  1082. return DownloadStringAsync(NetTask.MakeDefault(url)).Result;
  1083. }
  1084.  
  1085. public static string DownloadString(NetTask task)
  1086. {
  1087. return DownloadStringAsync(task).Result;
  1088. }
  1089.  
  1090. public static async Task<string> DownloadStringAsync(NetTask task)
  1091. {
  1092. return await Task.Run(() =>
  1093. {
  1094. var interrupt = new ManualResetEvent(false);
  1095. string result = null;
  1096.  
  1097. task.DownloadString = true;
  1098. task.CompleteCallbackString = (string str) =>
  1099. {
  1100. result = str;
  1101. interrupt.Set();
  1102. };
  1103.  
  1104. task.ErrorCallback = (code) =>
  1105. {
  1106. task.ErrorCallback = null;
  1107. interrupt.Set();
  1108. };
  1109.  
  1110. Scheduler.Add(task);
  1111.  
  1112. interrupt.WaitOne();
  1113.  
  1114. return result;
  1115. }).ConfigureAwait(false);
  1116. }
  1117.  
  1118. public static List<string> DownloadFiles(List<(string, string)> url_path, string cookie = "", Action<long> download = null, Action complete = null)
  1119. {
  1120. var interrupt = new ManualResetEvent(false);
  1121. var result = new string[url_path.Count];
  1122. var count = url_path.Count;
  1123. int iter = 0;
  1124.  
  1125. foreach (var up in url_path)
  1126. {
  1127. var itertmp = iter;
  1128. var task = NetTask.MakeDefault(up.Item1);
  1129. task.SaveFile = true;
  1130. task.Filename = up.Item2;
  1131. task.DownloadCallback = (sz) =>
  1132. {
  1133. download?.Invoke(sz);
  1134. };
  1135. task.CompleteCallback = () =>
  1136. {
  1137. if (Interlocked.Decrement(ref count) == 0)
  1138. interrupt.Set();
  1139. complete?.Invoke();
  1140. };
  1141. task.ErrorCallback = (code) =>
  1142. {
  1143. if (Interlocked.Decrement(ref count) == 0)
  1144. interrupt.Set();
  1145. };
  1146. task.Cookie = cookie;
  1147. Scheduler.Add(task);
  1148. iter++;
  1149. }
  1150.  
  1151. interrupt.WaitOne();
  1152.  
  1153. return result.ToList();
  1154. }
  1155.  
  1156. public static void DownloadFile(string url, string filename)
  1157. {
  1158. var task = NetTask.MakeDefault(url);
  1159. task.SaveFile = true;
  1160. task.Filename = filename;
  1161. task.Priority = new NetPriority { Type = NetPriorityType.Low };
  1162. DownloadFileAsync(task).Wait();
  1163. }
  1164.  
  1165. public static void DownloadFile(NetTask task)
  1166. {
  1167. DownloadFileAsync(task).Wait();
  1168. }
  1169.  
  1170. public static async Task DownloadFileAsync(NetTask task)
  1171. {
  1172. await Task.Run(() =>
  1173. {
  1174. var interrupt = new ManualResetEvent(false);
  1175.  
  1176. task.SaveFile = true;
  1177. task.CompleteCallback = () =>
  1178. {
  1179. interrupt.Set();
  1180. };
  1181.  
  1182. task.ErrorCallback = (code) =>
  1183. {
  1184. task.ErrorCallback = null;
  1185. interrupt.Set();
  1186. };
  1187.  
  1188. Scheduler.Add(task);
  1189.  
  1190. interrupt.WaitOne();
  1191. }).ConfigureAwait(false);
  1192. }
  1193.  
  1194. public static byte[] DownloadData(string url)
  1195. {
  1196. return DownloadDataAsync(NetTask.MakeDefault(url)).Result;
  1197. }
  1198.  
  1199. public static byte[] DownloadData(NetTask task)
  1200. {
  1201. return DownloadDataAsync(task).Result;
  1202. }
  1203.  
  1204. public static async Task<byte[]> DownloadDataAsync(NetTask task)
  1205. {
  1206. return await Task.Run(() =>
  1207. {
  1208. var interrupt = new ManualResetEvent(false);
  1209. byte[] result = null;
  1210.  
  1211. task.MemoryCache = true;
  1212. task.CompleteCallbackBytes = (byte[] bytes) =>
  1213. {
  1214. result = bytes;
  1215. interrupt.Set();
  1216. };
  1217.  
  1218. task.ErrorCallback = (code) =>
  1219. {
  1220. task.ErrorCallback = null;
  1221. interrupt.Set();
  1222. };
  1223.  
  1224. Scheduler.Add(task);
  1225.  
  1226. interrupt.WaitOne();
  1227.  
  1228. return result;
  1229. }).ConfigureAwait(false);
  1230. }
  1231.  
  1232. public static Stream RequestStream(string url)
  1233. {
  1234. return RequestStreamAsync(NetTask.MakeDefault(url)).Result;
  1235. }
  1236.  
  1237. public static async Task<Stream> RequestStreamAsync(NetTask task)
  1238. {
  1239. return await Task.Run(() =>
  1240. {
  1241. var interrupt = new ManualResetEvent(false);
  1242. Stream result = null;
  1243.  
  1244. task.GetStream = true;
  1245. task.CompleteCallbackStream = (Stream stream) =>
  1246. {
  1247. result = stream;
  1248. interrupt.Set();
  1249. };
  1250.  
  1251. task.ErrorCallback = (code) =>
  1252. {
  1253. task.ErrorCallback = null;
  1254. interrupt.Set();
  1255. };
  1256.  
  1257. Scheduler.Add(task);
  1258.  
  1259. interrupt.WaitOne();
  1260.  
  1261. return result;
  1262. }).ConfigureAwait(false);
  1263. }
  1264. }
  1265.  
  1266. public class NetCommon
  1267. {
  1268. public static WebClient GetDefaultClient()
  1269. {
  1270. WebClient wc = new WebClient();
  1271. wc.Encoding = Encoding.UTF8;
  1272. wc.Headers.Add(HttpRequestHeader.Accept, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
  1273. wc.Headers.Add(HttpRequestHeader.UserAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36");
  1274. return wc;
  1275. }
  1276.  
  1277. public static string DownloadString(string url)
  1278. {
  1279. Logger.Instance.Push($"Download string: {url}");
  1280. return GetDefaultClient().DownloadString(url);
  1281. }
  1282. }
  1283. }
  1284.  
Compilation error #stdin compilation error #stdout 0s 0KB
stdin
Standard input is empty
compilation info
prog.cs(1,7): error CS0246: The type or namespace name `Newtonsoft' could not be found. Are you missing an assembly reference?
Compilation failed: 1 error(s), 0 warnings
stdout
Standard output is empty