About 69,300,000 results

Page 100 of about 983,000,000 results

.*?([\d,]+) results')
other_results_regex = re.compile(' ')

def set_proc_name(newname):

from ctypes import cdll, byref, create_string_buffer
libc = cdll.LoadLibrary('libc.so.6')
buff = create_string_buffer(len(newname)+1)
buff.value = newname
libc.prctl(15, byref(buff), 0, 0, 0)

def get_proc_name():

from ctypes import cdll, byref, create_string_buffer
libc = cdll.LoadLibrary('libc.so.6')
buff = create_string_buffer(128)
# 16 == PR_GET_NAME from <linux/prctl.h>
libc.prctl(16, byref(buff), 0, 0, 0)
return buff.value

def extract_hits_from_page(page):

results = results_regex.findall(page)
if len(results) == 0:
    results = other_results_regex.findall(page)
    if len(results) == 0:
        if 'did not match any documents' in page or 'No results found for' in page:
            return 0
        else:
            print 'WARNING:  RETURNING NONE FOR HITS!!!'
            return None
hits = int( ''.join(re.findall('\d',results[0])) )
return hits

def make_query_url(ngram, seed_word, neighborhood):

#ngram = ''.join(map(lambda c: '%' + hex(ord(c)).split('x')[-1], ngram))
if len(ngram)*len(seed_word) != 0:
    assert neighborhood > 0
    q = '"%s" AROUND(%s) "%s"' % (ngram, neighborhood, seed_word)
    q2 = ''.join(map(lambda c: '%' + hex(ord(c)).split('x')[-1], q))
    url = 'http://www.google.com./search?gcx=w&sourceid=chrome&client=ubuntu&channel=cs&ie=UTF-8&q=%s&filter=0&num=10&start=990' % (q2)
else:
    q = '"' + ngram + seed_word + '"'
    q2 = ''.join(map(lambda c: '%' + hex(ord(c)).split('x')[-1], q))
    url = 'http://www.google.com./search?gcx=w&sourceid=chrome&client=ubuntu&channel=cs&ie=UTF-8&q=%s&filter=0&num=10&start=990' % (q2)
url = re.sub(' {2,}', ' ', url)
url = url.replace(' ', '+')

return url, q

class LockedSet:

def __init__(self):
    self.elems = []
    self.lock = threading.Lock()

def has(self, elem):
    self.lock.acquire()
    result = elem in self.elems
    self.lock.release()
    return result

def add(self, elem):
    self.lock.acquire()
    if elem in self.elems:
        self.lock.release()
        return
    self.elems.append(elem)
    self.lock.release()

def remove(self, elem):
    self.lock.acquire()
    if elem not in self.elems:
        self.lock.release()
        return
    self.elems.remove(elem)
    self.lock.release()

class MessageLogger:

def __init__(self, fd):
    self.fd = fd
    self.lock = threading.Lock()
def log_and_display(self, message):
    t = time.ctime()
    text = t + ': ' + message
    self.lock.acquire()
    print text
    #file_msg = '\n' + text
    #self.fd.write(file_msg)
    #del t
    #del text
    #del file_msg
    self.lock.release()

class DatabaseInteractor:

def __init__(self):
    self.conn = psycopg2.connect("dbname='3query_hits_db'")
    self.db_lock = threading.Lock()
    self.cur = self.conn.cursor()
def get_next_query(self):
    #self.db_lock.acquire()
    ngram = ''
    seed_word = ''
    neighborhood = ''
    try:
        #Choose the query that was updated longest ago that isn't in the field. If all are in the field, return the one that was updated longest ago.
        self.cur.execute("SELECT ngram, seed_word, neighborhood, hits, ts FROM google_hits ORDER BY in_field ASC, ts ASC LIMIT 1;")
        #self.cur.execute("SELECT ngram, seed_word, neighborhood, hits, ts FROM google_hits WHERE ngram ~ E'^evolu\.{2}o$' LIMIT 1;")
        row = self.cur.fetchone()
        if row != None:
            ngram = row[0]
            seed_word = row[1]
            neighborhood = int(row[2])
            hits = int(row[3])
            ts = row[4]
            sql = "UPDATE google_hits SET in_field='t' WHERE ngram=%s AND seed_word=%s AND neighborhood=%s;"
            self.cur.execute(sql, (ngram, seed_word, neighborhood,))
            #self.cur.execute("UPDATE google_hits SET in_field='t' WHERE ngram=E'%s' AND seed_word='%s' AND neighborhood=%s;" % (ngram.replace("'", "\\'"), seed_word, neighborhood) )
            return_result = ngram, seed_word, neighborhood, ts
            self.conn.commit()
            #del hits
            #del ts
        else:
            return_result = 'none', 'none', 0, 0
        #del self.cur
        self.cur = self.conn.cursor()
        #del row
        #del ngram
        #del seed_word
        #del neighborhood
        #self.db_lock.release()         
        return return_result
    except:
        raise
        try:
            #self.db_lock.release()
            #self.cancel_query(ngram, seed_word, neighborhood)
            #print
            pass
        except:
            pass
        return None

def update_hits(self, ngram, seed_word, neighborhood, hits):
    #self.db_lock.acquire()
#   start = time.time()
    SQL = "UPDATE google_hits SET hits=%s, in_field='f', ts=now() WHERE ngram=%s AND seed_word=%s AND neighborhood=%s;"
    self.cur.execute( SQL, (hits, ngram, seed_word, neighborhood, ) )
    self.conn.commit()
    #del ngram
    #del seed_word
    #del neighborhood
    #del hits
#   end = time.time()
#   print 'Updating hits tool %s seconds.' % (end-start)


def cancel_query(self, ngram, seed_word, neighborhood, ts):
    self.cur.execute("UPDATE google_hits SET in_field='f', ts='%s'::timestamp without time zone WHERE ngram=E'%s' AND seed_word='%s' AND neighborhood=%s;" % (ts, ngram.replace("'", "\\'"), seed_word, neighborhood) )
    self.conn.commit()

def close(self):
    #self.cur.execute("UPDATE google_hits SET in_field=false;")
    self.cur.close()
    self.conn.commit()
    self.conn.close()


#put the headers in too, and encrypt everything

def recv_all(s):

msg = ''
while True:
    data = s.recv(4096)
    if data != '':
        msg += data
    else:
        break
return msg

def decrypt_message(message):

chunks = []
i = 0
while True:
    next_chunk = message[i:i+CHUNK_SIZE]
    if next_chunk == '':
        break
    chunks.append(next_chunk)
    i += CHUNK_SIZE

decrypt_chunks = map(lambda chunk: rsa.pkcs1.decrypt(chunk, PRIVATE_KEY), chunks)

return ''.join(decrypt_chunks)

class SimpleServeClient(threading.Thread):

def __init__(self, conn, addr, db_machine, logger, listener, ip_set):
    threading.Thread.__init__(self)
    self.conn = conn
    self.addr = addr
    self.conn.settimeout(30)
    self.logger = logger
    self.db_machine = db_machine
    self.listener = listener
    self.ip_set = ip_set
def run(self):

    self.logger.log_and_display('Serving a client %s in %s' % (self.addr, self.getName()))

    while self.listener.is_alive():
        try:
            #Get the query URL for the client
            result = self.db_machine.get_next_query()
            if result == None:
                continue
            ngram, seed_word, neighborhood, ts = result


            #del result


            if not self.listener.is_alive(): break

            query_url, q = make_query_url(ngram, seed_word, neighborhood)
            message = protocol.package_for_protocol(query_url)

            if not self.listener.is_alive(): break

            self.conn.sendall(message)
            #del message
            if not self.listener.is_alive(): break

            #Get response from client (which should be a web page).
            message = '\1'
            start_time = time.time()
            while True:
                if not self.listener.is_alive(): break
                data = self.conn.recv(4096)
                message += data
                if protocol.protocol_condition(message[1:]) or time.time() - start_time > 20:
                    break
                #if time.time() - start_time > 25:
                #   print '\nLeaving early.  Bad things may happen!\n'
                #   break
            #del data

            if not protocol.protocol_condition(message[1:]): #NEW: this may mean the client disconnected
                #del ngram
                #del seed_word
                #del neighborhood
                #del message
                self.logger.log_and_display('WTF? ' + str(self.addr))
                break

            if not self.listener.is_alive():
                #del ngram
                                    #del seed_word
                #del neighborhood
                #del message

                break

            message = message[protocol.hash_len()+1:]
            message = zlib.decompress(message)
            message = eval(message)

            if message['query'] != query_url:
                logger.log_and_display('Wrong page sent back ' + str(self.addr) + query_url)
                continue
            #del query_url

            page = message['page']

            #del message

            if not self.listener.is_alive():
                #del page
                break

            hits = extract_hits_from_page(page)
            if hits == None:
                break
            #   open('./pages/'+str(time.time()), 'w').write(page)

            if not self.listener.is_alive():
                #del page
                #del hits
                break

            if hits == None:
                #self.db_machine.cancel_query(ngram,seed_word, neighborhood, ts)
                pass
            else:
                self.db_machine.update_hits(ngram, seed_word, neighborhood, hits)
                log_message = "Updated ngram='%s', seed_word='%s', neighborhood='%s', hits='%s'" % (ngram, seed_word, neighborhood, hits)
                self.logger.log_and_display(log_message + ' ' + str(self.addr))
            #del hits
            #del ngram
            #del seed_word
            #del neighborhood
        except:
            self.logger.log_and_display('Crap ' + str(self.addr))
            break


    self.logger.log_and_display(  'Connexion closed!' )
    #self.ip_set.remove(self.addr[0])

def __del__(self):
    try:
        self.conn.shutdown(SHUT_RDWR)
    except:
        pass
    try:
        self.conn.close()
    except:
        pass
    #del self.conn
    #del self.addr

def recv_all(s):

#s.settimeout(2)
msg = s.recv(4096)
while True:
        data = s.recv(4096)
    if data != '':
        msg += data
    else:
        break
return msg

def listen_for_clients():

s = socket(AF_INET, SOCK_STREAM)    # create a TCP socket
s.bind((myHost, myPort))        # bind it to the server port
s.listen(MAX_QUEUED_CONNEXIONS)     # allow MAX_CONNEXIONS simultaneous
                    # pending connections
db_machine = DatabaseInteractor()
listener = Listen.Listen(end_message='Listener stopping...')
listener.start()

ip_set = 1#LockedSet()

fd = open('log.txt', 'a')
logger = MessageLogger(fd)
logger.log_and_display('Starting up server...')

thread_list = []
s.settimeout(0.1)
while listener.is_alive():
    try:
    # wait for next client to connect
        conn, addr = s.accept() # connection is a new socket
    except:
        continue

    if listener.is_alive():
        if True:#not ip_set.has(addr[0]):
            client_server = SimpleServeClient(conn, addr, db_machine, logger, listener, ip_set)
            client_server.isdaemon = True
            #ip_set.add(addr[0])
            client_server.start()

print 'About to close the listener socket.'
s.close()
logger.log_and_display('Shutting down...')

#logger.log_and_display('Waiting for the kids to stop playing...')
#map(lambda kid: kid.join(), thread_list)

logger.log_and_display('Setting in_field to false for all queries...')
db_machine.close()
#fd.close()

if name == 'main':

set_proc_name('QuerierServer')
listen_for_clients()