-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathconn_client.py
111 lines (99 loc) · 3.8 KB
/
conn_client.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
import json
import time
from msg_socket import MsgSocket
# define request command strings
__CMD_CONNECT__ = "connect"
__CMD_TASK__ = "task"
__CMD_XPATH__ = "xpath"
__CMD_SUBMIT__ = "submit"
__CMD_EXIT__ = "exit"
# define connection status codes
__CODE_CONN_SUCCESS__ = 100
__CODE_CONN_FAILED__ = 101
__CODE_TASK_SUCCESS__ = 200
__CODE_TASK_FAILED__ = 201
__CODE_TASK_EMPTY__ = 210
__CODE_XPATH_SUCCESS__ = 300
__CODE_XPATH_FAILED__ = 301
__CODE_SUBMIT_SUCCESS__ = 400
__CODE_SUBMIT_FAILED__ = 401
__CODE_SUBMIT_EXIT__ = 410
# define connection macros
__BUFF_SIZE__ = 4096
class ClientConnection:
__msgsock__ = None
__client_id__ = None
__flag_connected__ = False
# init the class and establish the socket connection
def __init__(self, client_id, address, port):
self.__client_id__ = client_id
self.__msgsock__ = MsgSocket()
while self.__msgsock__.connect_ex(address, port) != 0:
print("server offline")
time.sleep(3)
# connect to the server and exchange information
# program will block here until the connection established
def connect(self):
if self.__flag_connected__:
raise ConnectionError("server already connected")
req = dict(id=self.__client_id__, cmd=__CMD_CONNECT__, data=None)
while True:
res = self.__request__(req)
if int(res["status"]) == __CODE_CONN_SUCCESS__:
break
else:
print("connection error")
time.sleep(3)
peer = self.__msgsock__.getpeername()
print("connected to {0} on port {1}".format(peer[0], peer[1]))
self.__flag_connected__ = True
# request the server for sending task(s)
# will return a list of urls
def request_tasks(self, count=1):
if not self.__flag_connected__:
raise ConnectionError("server not connected")
req = dict(id=self.__client_id__, cmd=__CMD_TASK__, data={"count": count})
while True:
res = self.__request__(req)
if int(res["status"]) == __CODE_TASK_SUCCESS__:
break
elif int(res["status"]) == __CODE_TASK_EMPTY__:
print("server task list empty")
time.sleep(3)
else:
print("task request error")
time.sleep(3)
return res["data"]["urls"]
# request the server for the xpath rules
# param host is host field in the url. eg: news.163.com
def request_xpath(self, host):
if not self.__flag_connected__:
raise ConnectionError("server not connected")
req = dict(id=self.__client_id__, cmd=__CMD_XPATH__, data={"host": host})
res = self.__request__(req)
if int(res["status"]) == __CODE_XPATH_SUCCESS__:
return res["data"][host]
else:
return None
# submit parsed data to the server
# param data must be an instance of list
def submit_data(self, data_list):
if not self.__flag_connected__:
raise ConnectionError("server not connected")
if type(data_list) is not list:
raise TypeError("param 'data_list' must be type of 'list'")
req = dict(id=self.__client_id__, cmd=__CMD_SUBMIT__, data={"artilist": data_list})
while True:
self.__msgsock__.send_msg(json.dumps(req))
res = json.loads(self.__msgsock__.recv_msg())
if int(res["status"]) == __CODE_SUBMIT_SUCCESS__:
return 0
elif int(res["status"]) == __CODE_SUBMIT_EXIT__:
return 1
else:
print("data submission error, retry after 3 seconds")
time.sleep(3)
def __request__(self, req):
self.__msgsock__.send_msg(json.dumps(req))
res = json.loads(self.__msgsock__.recv_msg())
return res