# !/usr/bin/env python

# encoding:UTF-8

from util import request_url

import re

import os

import sys

#from __future__ import print_function

from pptx import Presentation

from pptx.util import Inches

import PIL

class Crawler(object):

def __init__(self):

self.main_url = "https://mp.weixin.qq.com/s?__biz=MzA3NzIwMDM3OA==&mid=209853452&idx=1&sn=bd40e9622dca2e5bd52af08bbf870861&pass_ticket=8MmcYuwV6RkFHjUHOnxmzVg%2FEhQYTM26Zg%2FO2ZpgJVGyL6ewBt5fJc%2BEsNkytOiN"

self.media_content_pattern = re.compile('<div class="rich_media_content " id="js_content">.*?</div>',re.S)

self.item_pattern = re.compile('<p><a href="(.*?)" target=',re.S)

self.title_pattern = re.compile('<h2 class="rich_media_title" id="activity-name">(.*?)</h2>',re.S)

self.elements_content_pattern = re.compile('<p style=.*?</p>',re.S)

self.png_pattern = re.compile('data-src="(.*?)"',re.S)

self.datatype_pattern = re.compile('data-type="(.*?)"',re.S)

self.text_pattern = re.compile('>(.*?)</',re.S)

self.picid_pattern = re.compile('http://mmbiz.qpic.cn/mmbiz/(.*?)/',re.S)

self.pic_fmt_pattern = re.compile('wx_fmt=(.*?)$',re.S)

self.data_path = "../data/"

self.ppt_path = "../ppt/"

def get_item_list(self):

ret,main_page = request_url(self.main_url)

if ret == -1 or main_page == "":

print "Request main page failed!"

return

info = self.media_content_pattern.findall(main_page);

if len(info) != 0:

media_content = info[0]

else:

media_content = ""

item_list = []

if media_content:

item_info = self.item_pattern.findall(main_page);

if len(item_info) != 0:

for item_url in item_info:

print item_url

item_list.append(item_url)

title = self.get_item(item_url)

if title== "" and item_url.find("&amp;")!=-1:

item_url = item_url.replace("&amp;","&")

title = self.get_item(item_url)

def get_item(self,item_url):

ret,item_page = request_url(item_url)

if ret == -1 or item_page == "":

print "Request item page failed! %s" % item_url

return "bad"

info = self.title_pattern.findall(item_page);

if len(info) != 0:

title = info[0].strip().replace("(ppt)","")

else:

title = ""

item_path = self.data_path+"bak"

ppt_file = self.ppt_path+"bak"

if title!="":

item_path = self.data_path + title

ppt_file = self.ppt_path+ title + ".pptx"

else:

print "title is null!%s" % item_url

return ""

if os.path.exists(ppt_file):

return "exist"

info = self.media_content_pattern.findall(item_page);

if len(info) != 0:

media_content = info[0]

else:

media_content = ""

if media_content == "":

return

info = self.elements_content_pattern.findall(media_content);

element_tuple_list = []

for element_content in info:

if element_content.find('data-src="http://')!=-1:

element_type = "png"

else:

element_type = "text"

element_data = ""

if element_type == "png":

info = self.png_pattern.findall(element_content);

if len(info) != 0:

element_data = info[0]

if element_data.find("wx_fmt")==-1:

info = self.datatype_pattern.findall(element_content)

if len(info)>0:

element_data += "?wx_fmt=%s" % info[0]

else:

element_data = ""

else:

info = self.text_pattern.findall(element_content);

if len(info)>3:

element_data = ""

else:

element_data = "\n".join(info)

if element_data:

element_tuple_list.append((element_type,element_data))

if len(element_tuple_list) > 0:

if not os.path.exists(item_path):

os.makedirs(item_path)

text_data = ""

picfile_list = []

for element_tuple in element_tuple_list:

element_type,element_data = element_tuple

if element_type == "text":

text_data += element_data

else:

if element_data:

picfile = self.download_pic(element_data,item_path)

if picfile!="":

picfile_list.append(picfile)

self.write_text_content(text_data,item_path)

self.creat_ppt(title,ppt_file,picfile_list)

return title

def download_pic(self,url,path):

ret,pic_page = request_url(url)

if ret == -1 or pic_page=="":

return ""

info = self.picid_pattern.findall(url)

if len(info)>0 and info[0]!="":

picid = info[0]

else:

picid = url.replace("/","_").replace(":","_").replace(".","_").replace("?","_")

info = self.pic_fmt_pattern.findall(url)

if len(info)>0 and info[0]!="":

fmt = info[0].split("&")[0]

fmt = fmt.split("?")[0]

else:

print "Get pic fmt failed!%s" % url

return ""

filename_bak = "../data/" + picid + "_bak.%s" % fmt

fp = open(filename_bak,"w")

fp.write(pic_page)

fp.close()

pil_image = PIL.Image.open(filename_bak)

w, h = pil_image.size

w_box = 720

h_box = 540

filename = path + "/" + picid + ".%s" % fmt

self.resize(w, h, w_box, h_box, pil_image,filename)

os.system("rm -rf %s" % filename_bak)

return filename

def resize(self,w, h, w_box, h_box, pil_image,outfile):

'''

resize a pil_image object so it will fit into

a box of size w_box times h_box, but retain aspect ratio

'''

f1 = 1.0*w_box/w # 1.0 forces float division in Python2

f2 = 1.0*h_box/h

factor = min([f1, f2])

#print(f1, f2, factor) # test

# use best down-sizing filter

width = int(w*factor)

height = int(h*factor)

out = pil_image.resize((width, height), PIL.Image.ANTIALIAS)

out.save(outfile)

def write_text_content(self,text_data,path):

filename = path + "/" + "text_data.txt"

fp = open(filename,"w")

fp.write(text_data)

fp.close()

def creat_ppt(self,title_content,ppt_file,picfile_list):

prs = Presentation("default.pptx")

title_slide_layout = prs.slide_layouts[0]

slide = prs.slides.add_slide(title_slide_layout)

title = slide.shapes.title

title.text = title_content

graph_slide_layout = prs.slide_layouts[6]

for picfile in picfile_list:

slide = prs.slides.add_slide(graph_slide_layout)

slide.shapes.add_picture(picfile,0,0)

prs.save(ppt_file)

def test_ppt():

picfile = "test.jpeg"

for i in xrange(12):

try:

prs = Presentation("default.pptx")

graph_slide_layout = prs.slide_layouts[i]

slide = prs.slides.add_slide(graph_slide_layout)

placeholder = slide.placeholders[0]

#pic = placeholder.insert_picture(picfile)

prs.save("../ppt/%s.pptx" % i)

except:

continue

def test_layout(i):

picfile = "test.jpeg"

prs = Presentation("default.pptx")

graph_slide_layout = prs.slide_layouts[i]

slide = prs.slides.add_slide(graph_slide_layout)

placeholder = slide.placeholders[1]

pic = placeholder.insert_picture(picfile)

prs.save("../ppt/%s.pptx" % i)

def test_empty_layout():

picfile = "test.jpeg"

prs = Presentation("default.pptx")

graph_slide_layout = prs.slide_layouts[6]

slide = prs.slides.add_slide(graph_slide_layout)

slide.shapes.add_picture(picfile,0,0)

prs.save("../ppt/%s.pptx" % 6)

if __name__=="__main__":

#test_empty_layout()

#test_layout(int(sys.argv[1]))

#test_ppt()

crawler = Crawler()

#crawler.get_item_list()

item_url = "http://mp.weixin.qq.com/s?__biz=MzA3NzIwMDM3OA==&amp;mid=206906414&amp;idx=1&amp;sn=484555cf9c8efd164d06f6f6d0a6c19e&amp;scene=21#wechat_redirect"

item_url = item_url.replace("&amp;","&")

crawler.get_item(item_url)

print "done!"

 

util.py

# !/usr/bin/env python

# encoding:UTF-8

import urllib2

def request_url(url,repeat=3):

ret = -1#失败

content = ''

for cnt in xrange(repeat):

try:

req = urllib2.Request(url);

response = urllib2.urlopen(req)

content = response.read()

response.close()

ret = 0#成功

break;

except:

continue

result = (ret,content)

return result