-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathProgram.cs
More file actions
123 lines (119 loc) · 4.91 KB
/
Copy pathProgram.cs
File metadata and controls
123 lines (119 loc) · 4.91 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
using System;
using System.Collections.Generic;
using System.Linq;
namespace s2._3
{
class Program
{
static void Runner(string lnk)
{
if (!gotArticles.Contains(lnk))
{
gotArticles.Add(lnk);
FileController.AppendLine(artCollection, lnk);
LentaArticle art;
try
{
art = new LentaArticle(lnk);
}
catch (Exception e)
{
WriteExecption(e, String.Format("Ошибка при получении статьи {0}.", lnk));
return;
}
var key = art.GetCategory();
if (!gotCategories.ContainsKey(key))
gotCategories.Add(key, 0);
if (gotCategories[key] < artCnt)
{
gotCategories[key]++;
WriteReceivedArticle(art, lnk);
FileController.RewriteCollectionDict(artCategories, gotCategories);
try
{
dbController.ArticleInsert(art.GetHead(), art.GetBody(), art.GetDateTime().ToString(), art.GetCategory());
var end = "";
foreach (var item in gotCategories)
if (item.Value < artCnt)
end += String.Format($"{item.Key}-{item.Value} ");
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine(end);
Console.ResetColor();
}
catch (Exception e)
{
WriteExecption(e, "Ошибка при добавлении записи в базу данных.");
}
foreach (var url in art.GetLinksNext())
Runner(url);
}
}
}
static void WriteExecption(Exception e, string text)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(e.Message + " " + text);
Console.ResetColor();
}
static void WriteReceivedArticle(LentaArticle art, string lnk)
{
Console.Write($"{gotArticles.Count()}. {art.GetHead()} - {art.GetCategory()} ");
Console.ForegroundColor = ConsoleColor.Green;
Console.Write(lnk + "\n\r");
Console.ResetColor();
}
static string GetPath()
{
Console.WriteLine("Введите директорию для сохранения данных (будет создана отдельная папка)");
string path = Console.ReadLine();
while (!System.IO.Directory.Exists(path))
{
Console.WriteLine("Указанная директория не существует");
path = Console.ReadLine();
}
path += "/lenta_articles_parser/";
System.IO.Directory.CreateDirectory(path);
return path;
}
static int GetArtCount()
{
Console.WriteLine("Введите количество статей по каждоый категориии или 0 если нужны все");
string input = Console.ReadLine();
int artCnt = -1;
while (!int.TryParse(input, out artCnt) || artCnt < 0)
{
Console.WriteLine("Некорректный ввод");
input = Console.ReadLine();
}
if (artCnt == 0)
artCnt = int.MaxValue;
return artCnt;
}
static Dictionary<string, int> gotCategories;
static HashSet<string> gotArticles;
static DBController dbController;
static int artCnt = 0;
static string artCategories = "";
static string artCollection = "";
static string DBpath = "";
static void Main(string[] args)
{
var path = GetPath();
artCnt = GetArtCount();
artCategories = path + "articlesCategories.txt";
artCollection = path + "articlesCollection.txt";
DBpath = path + "articles.db";
gotCategories = FileController.GetCollectionDicti(artCategories);
gotArticles = FileController.GetHashSet(artCollection);
dbController = new DBController(DBpath);
var baseUrl = "https://lenta.ru/";
for (DateTime dt = DateTime.Now; dt.Year >= 2001; dt = dt.AddDays(-1))
{
Runner(baseUrl + dt.Year.ToString()
+ "/" + String.Format("{0:d2}", dt.Month)
+ "/" + String.Format("{0:d2}", dt.Day)
+ "/");
}
}
}
}