19955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard#!/usr/bin/python -u 29955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard# 39955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard# Processing of the queries results 49955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard# 59955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardimport sys 69955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardimport index 79955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardimport time 89955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardimport traceback 99955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardimport string 109955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 119955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardif index.openMySQL(verbose = 0) < 0: 129955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard print "Failed to connect to the MySQL database" 139955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard sys.exit(1) 149955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 159955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel VeillardDB = index.DB 169955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 179955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef getTopQueriesDB(base = "Queries", number = 50): 189955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard global DB 199955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 209955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard try: 219955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard import os 229955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard os.mkdir("searches") 239955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard except: 249955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard pass 259955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 269955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard date = time.strftime("%Y%m%d") 279955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard f = open("searches/%s-%s.xml" % (base, date), "w") 289955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard c = DB.cursor() 299955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard try: 309955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard ret = c.execute("""select sum(Count) from %s""" % (base)) 319955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard row = c.fetchone() 329955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard total = int(row[0]) 339955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard ret = c.execute("""select count(*) from %s""" % (base)) 349955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard row = c.fetchone() 359955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard uniq = int(row[0]) 369955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard ret = c.execute( 379955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard """select * from %s ORDER BY Count DESC LIMIT %d""" % (base, number)) 389955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard i = 0; 399955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard f.write("<queries total='%d' uniq='%d' nr='%d' date='%s'>\n" % ( 409955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard total, uniq, number, date)) 419955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard while i < ret: 429955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard row = c.fetchone() 439955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard f.write(" <query count='%d'>%s</query>\n" % (int(row[2]), row[1])) 449955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard i = i + 1 459955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard f.write("</queries>\n") 469955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard except: 479955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard print "getTopQueries %s %d failed\n" % (base, number) 489955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard print sys.exc_type, sys.exc_value 499955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard return -1 509955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard f.close() 519955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 529955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef getTopQueries(number = 50): 539955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard return getTopQueriesDB(base = "Queries", number = number) 549955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 559955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef getAllTopQueries(number = 50): 569955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard return getTopQueriesDB(base = "AllQueries", number = number) 579955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 589955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef increaseTotalCount(Value, count): 599955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard global DB 609955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 619955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard c = DB.cursor() 629955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard try: 639955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard ret = c.execute("""select ID,Count from AllQueries where Value='%s'""" % 649955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard (Value)) 659955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard row = c.fetchone() 669955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard id = row[0] 679955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard cnt = int(row[1]) + count 689955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard ret = c.execute("""UPDATE AllQueries SET Count = %d where ID = %d""" % 699955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard (cnt, id)) 709955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard except: 719955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard ret = c.execute( 729955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard """INSERT INTO AllQueries (Value, Count) VALUES ('%s', %d)""" % 739955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard (Value, count)) 749955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 759955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 769955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef checkString(str): 779955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard if string.find(str, "'") != -1 or \ 789955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard string.find(str, '"') != -1 or \ 799955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard string.find(str, "\\") != -1 or \ 809955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard string.find(str, " ") != -1 or \ 819955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard string.find(str, "\t") != -1 or \ 829955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard string.find(str, "\n") != -1 or \ 839955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard string.find(str, "\r") != -1: 849955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard return 0 859955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard return 1 869955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef addCounts(frmtable): 879955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard global DB 889955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 899955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard i = 0 909955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard c = DB.cursor() 919955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard entries=[] 929955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard try: 939955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard ret = c.execute("""select Value,Count from %s""" % (frmtable)) 949955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard while i < ret: 959955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard i = i + 1 969955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard row = c.fetchone() 979955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard if checkString(row[0]): 989955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard entries.append((row[0], int(row[1]))) 999955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard else: 1009955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard entries.append((None, int(row[1]))) 1019955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 1029955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard for row in entries: 1039955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard if row[0] != None: 1049955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard increaseTotalCount(row[0], row[1]) 1059955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard except: 1069955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard print "addCounts %s failed" % (frmtable) 1079955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard print sys.exc_type, sys.exc_value 1089955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard traceback.print_exc(file=sys.stdout) 1099955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 1109955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard try: 1119955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard c.execute("""DELETE from %s""" % (frmtable)) 1129955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard except: 1139955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard pass 1149955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 1159955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 1169955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard 1179955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel VeillardgetTopQueries() 1189955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel VeillardaddCounts('Queries') 1199955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel VeillardgetAllTopQueries() 120