python 中文字数统计/分词

1、linux/GNU => debian/ubuntu 12.04/Linuxmint 13Preferred
2、python
3、中文分词API， 这里我们使用的是 http://www.vapsec.com/fenci/
4、分词属性的说明文件下载 http://vdisk.weibo.com/s/qR7KSFDa9ON 或者 http://ishare.iask.sina.com.cn/f/68191875.html

#!/usr/bin/env python#coding: utf-8
importsys
importurllib
importurllib2
importos
importre
from datetime importdatetime, timedelta


def url_post(word='My name is Jake Anderson', geshi="json"):
    url = "http://open.vapsec.com/segment/get_word"postDict ={
            "word":word,
            "format":geshi
    }
    
    
    postData =urllib.urlencode(postDict)
    request =urllib2.Request(url, postData)
    request.get_method = lambda : 'POST'
    #request.add_header('Authorization', basic)
    response =urllib2.urlopen(request)
    r =response.readlines()
    printr

if __name__ == "__main__":
    f = open('novel2.txt', 'r')
    #get Chinese characters quantity
    regex=re.compile(r"(?x) (?: [w-]+ | [x80-xff]{3} )")
    count =0
    for line inf:
        line = line.decode('gbk')
        line = line.encode('utf8')
        word = [w for w inregex.split(line)]
        count +=len(word)
    #print count
    f = open('novel2.txt', 'r')
    start_time =datetime.now()
    for line inf:
        line = line.decode('gbk')
        line = line.encode('utf8')
        word2 = [w for w inregex.split(line)]
        printline
        url_post(line)
    end_time =datetime.now()
    tdelta = start_time -end_time
    print "It takes " + str(tdelta.total_seconds()) + "seconds to segment " + str(count) + "Chinese words!"
    print "This means it can segment " + str(count/tdelta.total_seconds()) + "Chinese characters per second!"