import urllib2
import random
#import sys
#reload(sys)
#sys.setdefaultencoding('utf-8')
user_agents = ['Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0', \
         'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0', \
         'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533+ \
         (KHTML, like Gecko) Element Browser 5.0', \
         'IBM WebExplorer /v0.94', 'Galaxy/1.0 [en] (Mac OS X 10.5.6; U; en)', \
         'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)', \
         'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14', \
         'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) \
         Version/6.0 Mobile/10A5355d Safari/8536.25', \
         'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) \
         Chrome/28.0.1468.0 Safari/537.36', \
         'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; TheWorld)']

def getDtd():

    print 'begin'
    fo = open('urls.txt', 'r')
    count = 1 
    while 1:
        urls = fo.readline()
        if not urls:
            break
        print urls
        page = getPage(urls)
        if len(page) > 0:
            file_name = urllib2.unquote(urls).decode('utf8').split('/')[-1]
            name=file_name.split('.')[0]
            #writeFile1(str(count)+'.'+str(name) , page)
            if page.find("<!DOCTYPE html")!=-1:#按照文件类型过滤
                writeFile2("docjar-"+str(count)+'.'+str(name), page)
                #writeFile2(str(name), page)
            else:
                writeFile1("docjar-"+str(count)+'.'+str(name), page)
            #writeFile2(file_name + str(count), page)
            fileFine=open('fine.txt','a')
            fileFine.write(urls)
            fileFine.close()
            print count
            count=count+ 1
            
        else:
            print count,'Error for linking'
            count=count+1
    print 'end'
def getPage(urls):
     req = urllib2.Request(urls)
     length = len(user_agents)
     index = random.randint(0, length-1)
     user_agent = user_agents[index] 
     req.add_header('User-agent', user_agent)
     try:
         resp = urllib2.urlopen(req)        
     except:
        print "Error"
        fileError=open('error.txt','a')
        fileError.write(urls)
        fileError.close()
        return ''
     pageinfo = resp.read()
     return pageinfo

def writeFile1(filename, content):
    fo = open(filename + '.dtd', 'w')
    fo.write(content)
    fo.close()
def writeFile2(filename, content):
    fo = open(filename + '.txt', 'w')
    fo.write(content)
    fo.close()

if __name__ == '__main__':
    getDtd()

