19955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard#!/usr/bin/python -u
29955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard#
39955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard# Processing of the queries results
49955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard#
59955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardimport sys
69955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardimport index
79955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardimport time
89955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardimport traceback
99955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardimport string
109955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
119955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillardif index.openMySQL(verbose = 0) < 0:
129955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    print "Failed to connect to the MySQL database"
139955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    sys.exit(1)
149955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
159955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel VeillardDB = index.DB
169955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
179955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef getTopQueriesDB(base = "Queries", number = 50):
189955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    global DB
199955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
209955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    try:
219955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        import os
229955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	os.mkdir("searches")
239955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    except:
249955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        pass
259955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
269955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    date = time.strftime("%Y%m%d")
279955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    f = open("searches/%s-%s.xml" % (base, date), "w")
289955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    c = DB.cursor()
299955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    try:
309955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        ret = c.execute("""select sum(Count) from %s""" % (base))
319955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	row = c.fetchone()
329955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	total = int(row[0])
339955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        ret = c.execute("""select count(*) from %s""" % (base))
349955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	row = c.fetchone()
359955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	uniq = int(row[0])
369955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        ret = c.execute(
379955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard           """select * from %s ORDER BY Count DESC LIMIT %d""" % (base, number))
389955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	i = 0;
399955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	f.write("<queries total='%d' uniq='%d' nr='%d' date='%s'>\n" % (
409955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	      total, uniq, number, date))
419955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	while i < ret:
429955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	    row = c.fetchone()
439955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	    f.write("  <query count='%d'>%s</query>\n" % (int(row[2]), row[1]))
449955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	    i = i + 1
459955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	f.write("</queries>\n")
469955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    except:
479955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        print "getTopQueries %s %d failed\n" % (base, number)
489955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	print sys.exc_type, sys.exc_value
499955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	return -1
509955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    f.close()
519955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
529955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef getTopQueries(number = 50):
539955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    return getTopQueriesDB(base = "Queries", number = number)
549955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
559955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef getAllTopQueries(number = 50):
569955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    return getTopQueriesDB(base = "AllQueries", number = number)
579955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
589955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef increaseTotalCount(Value, count):
599955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    global DB
609955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
619955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    c = DB.cursor()
629955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    try:
639955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        ret = c.execute("""select ID,Count from AllQueries where Value='%s'""" %
649955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	                (Value))
659955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	row = c.fetchone()
669955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	id = row[0]
679955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	cnt = int(row[1]) + count
689955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	ret = c.execute("""UPDATE AllQueries SET Count = %d where ID = %d""" %
699955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	                (cnt, id))
709955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    except:
719955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        ret = c.execute(
729955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	"""INSERT INTO AllQueries (Value, Count) VALUES ('%s', %d)""" %
739955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	                (Value, count))
749955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
759955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
769955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef checkString(str):
779955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    if string.find(str, "'") != -1 or \
789955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard       string.find(str, '"') != -1 or \
799955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard       string.find(str, "\\") != -1 or \
809955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard       string.find(str, " ") != -1 or \
819955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard       string.find(str, "\t") != -1 or \
829955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard       string.find(str, "\n") != -1 or \
839955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard       string.find(str, "\r") != -1:
849955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        return 0
859955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    return 1
869955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillarddef addCounts(frmtable):
879955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    global DB
889955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
899955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    i = 0
909955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    c = DB.cursor()
919955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    entries=[]
929955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    try:
939955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        ret = c.execute("""select Value,Count from %s""" % (frmtable))
949955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	while i < ret:
959955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	    i = i + 1
969955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	    row = c.fetchone()
979955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	    if checkString(row[0]):
989955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard		entries.append((row[0], int(row[1])))
999955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	    else:
1009955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard		entries.append((None, int(row[1])))
1019955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
1029955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	for row in entries:
1039955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	    if row[0] != None:
1049955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard		increaseTotalCount(row[0], row[1])
1059955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    except:
1069955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard        print "addCounts %s failed" % (frmtable)
1079955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	print sys.exc_type, sys.exc_value
1089955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	traceback.print_exc(file=sys.stdout)
1099955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
1109955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    try:
1119955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	c.execute("""DELETE from %s""" % (frmtable))
1129955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard    except:
1139955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard	pass
1149955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
1159955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
1169955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel Veillard
1179955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel VeillardgetTopQueries()
1189955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel VeillardaddCounts('Queries')
1199955d2b4a11a6a7c5607af37cd5932af5916e17cDaniel VeillardgetAllTopQueries()
120