EasyGoSpider/launch.py at master · ShanGis/EasyGoSpider · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
# coding: utf-8

import sys
import time
import math
import os
import sched
import argparse
from scrapy import cmdline


parser = argparse.ArgumentParser(description="EasyGoSpider")
group = parser.add_mutually_exclusive_group()
group.add_argument("--now", action="store_true",
                   help="Launch right now and only once.")
group.add_argument("--loop", action="store_true",
                   help="Launch once every two hours from next even o'clock on.")
args = parser.parse_args()
schedule = sched.scheduler(time.time, time.sleep)


def next_time(t):
    next_t = list(t)
    if math.fmod(next_t[3], 2) == 0:
        next_t[3] += 2
    else:
        next_t[3] += 1
    next_t = next_t[:4] + [0 for i in range(5)]
    return next_t


def perform_command(cmd, inc):
    schedule.enter(inc, 0, perform_command, (cmd, inc))
    os.system(cmd)


def timming_exe(cmd, inc):
    schedule.enter(inc, 0, perform_command, (cmd, 7200))
    schedule.run()


def loop():
    now = time.localtime()
    # logger.info("current time: %s" % now)
    next_t = next_time(now)
    # logger.info("...will be started after %s" % next_t)
    interval = time.mktime(next_t) - time.mktime(now) + 10
    # logger.info("...wait for %s seconds" % interval)
    timming_exe('scrapy crawl proc', interval)


if __name__ == '__main__':
    # cmdline.execute('scrapy crawl proc'.split())
    if len(sys.argv) == 1:
        parser.print_help()
    elif args.now:
        cmdline.execute('scrapy crawl proc'.split())
    elif args.loop:
        loop()