forked from nopriest/IPProxyPool
-
Notifications
You must be signed in to change notification settings - Fork 0
/
IPProxy.py
28 lines (24 loc) · 764 Bytes
/
IPProxy.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# coding:utf-8
from multiprocessing import Value, Queue, Process
from api.apiServer import start_api_server
from db.DataStore import store_data
from validator.Validator import validator, getMyIP
from spider.ProxyCrawl import startProxyCrawl
from config import TASK_QUEUE_SIZE
if __name__ == "__main__":
myip = getMyIP()
DB_PROXY_NUM = Value('i', 0)
q1 = Queue(maxsize=TASK_QUEUE_SIZE)
q2 = Queue()
p0 = Process(target=start_api_server)
p1 = Process(target=startProxyCrawl, args=(q1, DB_PROXY_NUM,myip))
p2 = Process(target=validator, args=(q1, q2, myip))
p3 = Process(target=store_data, args=(q2, DB_PROXY_NUM))
p0.start()
p1.start()
p2.start()
p3.start()
p0.join()
p1.join()
p2.join()
p3.join()