procrastium/souper.py at master · pythonfoo/procrastium · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
__author__ = 'bison'

import urllib
#import BeautifulSoup4
import requests
from bs4 import BeautifulSoup
import os
import sys
import datetime
import json

class souperDuper(object):
    def __init__(self, soupAccount):
        self.account = soupAccount
        self.counter = 0
        #self.knownUrlsFile = os.path.join("images", self.account + ".knownUrls.txt")
        self.knownUrlsFile = self.account + ".knownUrls.txt"
        self.knownUrls = {}

        self.lastPage = ""
        self.doResume = False
        self.imageTypes = ['.jpg', '.jpeg', '.gif']

    def loadKnownUrls(self, fullPath=""):
        if fullPath == "":
            fullPath = self.knownUrlsFile

        if os.path.isfile(fullPath):
            fh = open(fullPath, 'r')
            tmpJson = fh.read()
            if tmpJson != "":
                self.knownUrls = json.loads(tmpJson)
            fh.close()

    def saveKnownUrls(self, fullPath=""):
        if fullPath == "":
            fullPath = self.knownUrlsFile

        fh = open(fullPath, 'w')
        fh.write(json.dumps(self.knownUrls))
        fh.close()

    def grabPart(self, max=1):
        self.loadKnownUrls()

        html = self.grabPage()

        foundNext = True
        cnt = 0
        while foundNext:
            self.saveKnownUrls()

            nextSegment = self.getNextEndless(html)
            if not nextSegment:
                foundNext = False
            else:
                self.debug("got: " + str(len(self.knownUrls)) + " next: " + nextSegment)
                html = self.grabPage(nextSegment)
            cnt += 1
            if cnt >= max:
                self.saveKnownUrls()
                yield None
                cnt = 0

        self.saveKnownUrls()


    def grabAll(self):
        self.loadKnownUrls()

        html = self.grabPage()

        foundNext = True
        while foundNext:
            self.saveKnownUrls()

            nextSegment = self.getNextEndless(html)
            if not nextSegment:
                foundNext = False
            else:
                self.debug("got: " + str(len(self.knownUrls)) + " next: " + nextSegment)
                html = self.grabPage(nextSegment)

        self.saveKnownUrls()

    def grabPage(self, urlExt=""):
        grabUrl = 'http://' + self.account + '.soup.io'
        prettyHtml = ""
        #SOUP.Endless.next_url
        if urlExt != "":
            grabUrl += "/" + urlExt

        try:
            response = requests.get(grabUrl)
            html = response.content
            #print(html)
            soup = BeautifulSoup(html)

            #print(soup.prettify())
            #exit()
            #for link in soup.findAll('a'):
            #    print(link.get('href'))

            for img in soup.findAll('img'):
                imgUrl = str(img.get('src'))
                if "asset" in imgUrl and not "square" in imgUrl:
                    if not imgUrl in self.knownUrls and self._isValidFile(imgUrl):
                        self.counter += 1
                        self.knownUrls[imgUrl] = self.counter

                        self.debug(str(self.counter) + ' > ' + imgUrl)
            prettyHtml = soup.prettify()
        except Exception as ex:
            self.debug("ERROR: " + grabUrl + "\n" + str(ex))

        return prettyHtml

    def _isValidFile(self, url):
        if not self.imageTypes:
            return True

        #fileExt = imgUrl.split('.')
        #fileExt = fileExt[len(fileExt)-1]
        fileName, fileExt = os.path.splitext(url)
        #print fileName,"#", fileExt, fileExt in self.imageTypes
        if fileExt.lower() in self.imageTypes:
            return True

        return False

    def getSaveFileName(self, url=""):
        #tmp = url.replace('http://', '')
        tmp = url.split('/')
        return str(self.counter) + '#' + tmp[len(tmp)-2] + "#" + tmp[len(tmp)-1]

    def getNextEndless(self, html):
        #http://user.soup.io/since/294388231
        pret = html.split("SOUP.Endless.next_url = '")

        if len(pret) == 2:
            part = pret[1].split('?')[0]
            return part
        return ""

    def debug(self, txt):
        print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S|"+txt))


if __name__ == "__main__":
    if len(sys.argv) > 1:
        sd = souperDuper(sys.argv)
        sd.grabAll()
    else:
        print("first parameter must be the user you want to dupe")