From ac1fd7dfeb835478eec6f2a4bf425612dc874aa3 Mon Sep 17 00:00:00 2001 From: Sam Ginn Date: Mon, 23 Jan 2017 01:03:04 -0800 Subject: [PATCH 1/5] fix file write bug with file not getting flushed --- community-server/server.py | 54 +++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/community-server/server.py b/community-server/server.py index 7369cb8..50ab3c2 100755 --- a/community-server/server.py +++ b/community-server/server.py @@ -147,19 +147,21 @@ def emit_structs(): # get the most recent 7 structs for fname in sorted([int(n[:-5]) for n in os.listdir(uid_folder)])[:7]: path = os.path.join(uid_folder, str(fname) + ".json") + try: + with open(path, 'r') as f: + lines = f.readlines() + if (len(lines) != 3): + continue - with open(path, 'r') as f: - lines = f.readlines() - if (len(lines) != 3): - continue + upvotes = json.loads(lines[0].strip()) + timestamp = json.loads(lines[1].strip()) + struct = json.loads(lines[2].strip()) - upvotes = json.loads(lines[0].strip()) - timestamp = json.loads(lines[1].strip()) - struct = json.loads(lines[2].strip()) - - score = score_struct(timestamp, len(upvotes)) - message = {"uid": uid, "id": str(fname), "score": score, "upvotes": [scrub_uid(up) for up in upvotes], "struct": struct} - emit("struct", message) + score = score_struct(timestamp, len(upvotes)) + message = {"uid": uid, "id": str(fname), "score": score, "upvotes": [scrub_uid(up) for up in upvotes], "struct": struct} + emit("struct", message) + except: + pass def emit_utterances(): @@ -204,6 +206,8 @@ def emit_utterances(): def log(message): """Logs the given message by writing it in the uid's JSON log file.""" + uid = session.uid if hasattr(session, 'uid') else "NULL_session" + path = os.path.join(LOG_FOLDER, session.uid + ".json") # Add a timestamp to the log @@ -291,24 +295,32 @@ def upvote(data): # Read the first line of the file to get the number of upvotes upvotes = [] score = 0 - with open(struct_path, 'r') as f: + with open(struct_path, 'r+') as f: upvotes = json.loads(f.readline().strip()) # If the user has not already upvoted this, add them if session.uid not in upvotes: upvotes.append(session.uid) - with open(struct_path, 'w') as fw: - fw.write(json.dumps(upvotes) + "\n") - timestamp = f.readline() - fw.write(timestamp) # rewrite the timestamp - fw.write(f.readline()) # rewrite the actual struct + timestamp = f.readline() + struct = f.readline() - score = score_struct(timestamp, len(upvotes)) + # reset file to top + f.seek(0) - # and then broadcast the new upvote to the room: - message = {"uid": data["uid"], "id": data["id"], "up": scrub_uid(session.uid), "score": score} - emit("upvote", message, broadcast=True, room="community") + # write file back with updated upvotes + f.write(json.dumps(upvotes) + "\n") + f.write(timestamp) # rewrite the timestamp + f.write(struct) # rewrite the actual struct + + f.truncate() # truncate to ensure flush appropriate + + # calculate score + score = score_struct(timestamp, len(upvotes)) + + # and then broadcast the new upvote to the room: + message = {"uid": data["uid"], "id": data["id"], "up": scrub_uid(session.uid), "score": score} + emit("upvote", message, broadcast=True, room="community") @socketio.on('log') From 26d873c64b8c5988ffdda10c2f8294965956e57d Mon Sep 17 00:00:00 2001 From: Sam Ginn Date: Mon, 23 Jan 2017 01:18:47 -0800 Subject: [PATCH 2/5] don't scrub uid when sending to client --- community-server/server.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/community-server/server.py b/community-server/server.py index 50ab3c2..e8889d1 100755 --- a/community-server/server.py +++ b/community-server/server.py @@ -42,8 +42,7 @@ A client can make the following socket requests: filename. - "upvote": {"uid": UID, "id": ID} a user can upvote another user's struct by passing in the struct's - uid (the scrub_uid() of the user who submitted it) and the id of - the struct itself. + uid and the id of the struct itself. - "share": {"struct": STRUCT} a user can share a struct and it will be saved to their directory @@ -125,19 +124,6 @@ def current_unix_time(): return int(time.time()) -def scrub_uid(uid): - """We don't want to reveal other users' full session_id's to other users, - so we hide it by just sending the first 8 characters. - - NB: We use the session_id as the sole source of truth for authentication - and logging purposes - if it was leaked, other turkers would be able to - impersonate users. - - TODO: Convert to using signed JWTs. - """ - return uid[:8] - - def emit_structs(): """Walk through the STRUCTS_FOLDER directory and read each struct and emit it to the user one by one.""" @@ -158,7 +144,7 @@ def emit_structs(): struct = json.loads(lines[2].strip()) score = score_struct(timestamp, len(upvotes)) - message = {"uid": uid, "id": str(fname), "score": score, "upvotes": [scrub_uid(up) for up in upvotes], "struct": struct} + message = {"uid": uid, "id": str(fname), "score": score, "upvotes": [up for up in upvotes], "struct": struct} emit("struct", message) except: pass @@ -188,7 +174,7 @@ def emit_utterances(): latest_5[earliest_idx] = file_info for (time, uid, path) in sorted(latest_5, key=lambda s: int(s[0]), reverse=True): - uid = scrub_uid(uid) + uid = uid utts = [] count = 0 for line in reverse_readline(path): @@ -252,7 +238,7 @@ def handle_share(data): current score of the struct and ID is the unique index (auto-incremented) of this particular struct..""" - user_structs_folder = os.path.join(STRUCTS_FOLDER, scrub_uid(session.uid)) + user_structs_folder = os.path.join(STRUCTS_FOLDER, session.uid) make_dir_if_necessary(user_structs_folder) names = os.listdir(user_structs_folder) new_struct_id = "1" @@ -270,7 +256,7 @@ def handle_share(data): f.write(json.dumps(data["struct"])) # the actual struct # Broadcast addition to the "community" room - message = {"uid": scrub_uid(session.uid), "id": new_struct_id, "score": score, "upvotes": [], "struct": data["struct"]} + message = {"uid": session.uid, "id": new_struct_id, "score": score, "upvotes": [], "struct": data["struct"]} emit("struct", message, broadcast=True, room="community") @@ -319,7 +305,7 @@ def upvote(data): score = score_struct(timestamp, len(upvotes)) # and then broadcast the new upvote to the room: - message = {"uid": data["uid"], "id": data["id"], "up": scrub_uid(session.uid), "score": score} + message = {"uid": data["uid"], "id": data["id"], "up": session.uid, "score": score} emit("upvote", message, broadcast=True, room="community") @@ -339,10 +325,10 @@ def handle_log(data): # If the message is an accept or define type, broadcast it to all # community-connected clients so they can update their display. if data["type"] == "accept": - emit("new_accept", {"uid": scrub_uid(session.uid), "query": data["msg"]["query"], "timestamp": current_unix_time()}, + emit("new_accept", {"uid": session.uid, "query": data["msg"]["query"], "timestamp": current_unix_time()}, broadcast=True, room="community") elif data["type"] == "define": - emit("new_define", {"uid": scrub_uid(session.uid), "defined": data["msg"]["defineAs"], "timestamp": current_unix_time()}, + emit("new_define", {"uid": session.uid, "defined": data["msg"]["defineAs"], "timestamp": current_unix_time()}, broadcast=True, room="community") From 8cec892fa5a5da7805d6a2d9a68ba7f079ba4048 Mon Sep 17 00:00:00 2001 From: Sam Ginn Date: Mon, 23 Jan 2017 01:03:04 -0800 Subject: [PATCH 3/5] fix file write bug with file not getting flushed --- community-server/server.py | 54 +++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/community-server/server.py b/community-server/server.py index 7369cb8..50ab3c2 100755 --- a/community-server/server.py +++ b/community-server/server.py @@ -147,19 +147,21 @@ def emit_structs(): # get the most recent 7 structs for fname in sorted([int(n[:-5]) for n in os.listdir(uid_folder)])[:7]: path = os.path.join(uid_folder, str(fname) + ".json") + try: + with open(path, 'r') as f: + lines = f.readlines() + if (len(lines) != 3): + continue - with open(path, 'r') as f: - lines = f.readlines() - if (len(lines) != 3): - continue + upvotes = json.loads(lines[0].strip()) + timestamp = json.loads(lines[1].strip()) + struct = json.loads(lines[2].strip()) - upvotes = json.loads(lines[0].strip()) - timestamp = json.loads(lines[1].strip()) - struct = json.loads(lines[2].strip()) - - score = score_struct(timestamp, len(upvotes)) - message = {"uid": uid, "id": str(fname), "score": score, "upvotes": [scrub_uid(up) for up in upvotes], "struct": struct} - emit("struct", message) + score = score_struct(timestamp, len(upvotes)) + message = {"uid": uid, "id": str(fname), "score": score, "upvotes": [scrub_uid(up) for up in upvotes], "struct": struct} + emit("struct", message) + except: + pass def emit_utterances(): @@ -204,6 +206,8 @@ def emit_utterances(): def log(message): """Logs the given message by writing it in the uid's JSON log file.""" + uid = session.uid if hasattr(session, 'uid') else "NULL_session" + path = os.path.join(LOG_FOLDER, session.uid + ".json") # Add a timestamp to the log @@ -291,24 +295,32 @@ def upvote(data): # Read the first line of the file to get the number of upvotes upvotes = [] score = 0 - with open(struct_path, 'r') as f: + with open(struct_path, 'r+') as f: upvotes = json.loads(f.readline().strip()) # If the user has not already upvoted this, add them if session.uid not in upvotes: upvotes.append(session.uid) - with open(struct_path, 'w') as fw: - fw.write(json.dumps(upvotes) + "\n") - timestamp = f.readline() - fw.write(timestamp) # rewrite the timestamp - fw.write(f.readline()) # rewrite the actual struct + timestamp = f.readline() + struct = f.readline() - score = score_struct(timestamp, len(upvotes)) + # reset file to top + f.seek(0) - # and then broadcast the new upvote to the room: - message = {"uid": data["uid"], "id": data["id"], "up": scrub_uid(session.uid), "score": score} - emit("upvote", message, broadcast=True, room="community") + # write file back with updated upvotes + f.write(json.dumps(upvotes) + "\n") + f.write(timestamp) # rewrite the timestamp + f.write(struct) # rewrite the actual struct + + f.truncate() # truncate to ensure flush appropriate + + # calculate score + score = score_struct(timestamp, len(upvotes)) + + # and then broadcast the new upvote to the room: + message = {"uid": data["uid"], "id": data["id"], "up": scrub_uid(session.uid), "score": score} + emit("upvote", message, broadcast=True, room="community") @socketio.on('log') From 794e827526f4091e7c971984d6c354165bff334f Mon Sep 17 00:00:00 2001 From: Sam Ginn Date: Mon, 23 Jan 2017 01:18:47 -0800 Subject: [PATCH 4/5] don't scrub uid when sending to client --- community-server/server.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/community-server/server.py b/community-server/server.py index 50ab3c2..e8889d1 100755 --- a/community-server/server.py +++ b/community-server/server.py @@ -42,8 +42,7 @@ A client can make the following socket requests: filename. - "upvote": {"uid": UID, "id": ID} a user can upvote another user's struct by passing in the struct's - uid (the scrub_uid() of the user who submitted it) and the id of - the struct itself. + uid and the id of the struct itself. - "share": {"struct": STRUCT} a user can share a struct and it will be saved to their directory @@ -125,19 +124,6 @@ def current_unix_time(): return int(time.time()) -def scrub_uid(uid): - """We don't want to reveal other users' full session_id's to other users, - so we hide it by just sending the first 8 characters. - - NB: We use the session_id as the sole source of truth for authentication - and logging purposes - if it was leaked, other turkers would be able to - impersonate users. - - TODO: Convert to using signed JWTs. - """ - return uid[:8] - - def emit_structs(): """Walk through the STRUCTS_FOLDER directory and read each struct and emit it to the user one by one.""" @@ -158,7 +144,7 @@ def emit_structs(): struct = json.loads(lines[2].strip()) score = score_struct(timestamp, len(upvotes)) - message = {"uid": uid, "id": str(fname), "score": score, "upvotes": [scrub_uid(up) for up in upvotes], "struct": struct} + message = {"uid": uid, "id": str(fname), "score": score, "upvotes": [up for up in upvotes], "struct": struct} emit("struct", message) except: pass @@ -188,7 +174,7 @@ def emit_utterances(): latest_5[earliest_idx] = file_info for (time, uid, path) in sorted(latest_5, key=lambda s: int(s[0]), reverse=True): - uid = scrub_uid(uid) + uid = uid utts = [] count = 0 for line in reverse_readline(path): @@ -252,7 +238,7 @@ def handle_share(data): current score of the struct and ID is the unique index (auto-incremented) of this particular struct..""" - user_structs_folder = os.path.join(STRUCTS_FOLDER, scrub_uid(session.uid)) + user_structs_folder = os.path.join(STRUCTS_FOLDER, session.uid) make_dir_if_necessary(user_structs_folder) names = os.listdir(user_structs_folder) new_struct_id = "1" @@ -270,7 +256,7 @@ def handle_share(data): f.write(json.dumps(data["struct"])) # the actual struct # Broadcast addition to the "community" room - message = {"uid": scrub_uid(session.uid), "id": new_struct_id, "score": score, "upvotes": [], "struct": data["struct"]} + message = {"uid": session.uid, "id": new_struct_id, "score": score, "upvotes": [], "struct": data["struct"]} emit("struct", message, broadcast=True, room="community") @@ -319,7 +305,7 @@ def upvote(data): score = score_struct(timestamp, len(upvotes)) # and then broadcast the new upvote to the room: - message = {"uid": data["uid"], "id": data["id"], "up": scrub_uid(session.uid), "score": score} + message = {"uid": data["uid"], "id": data["id"], "up": session.uid, "score": score} emit("upvote", message, broadcast=True, room="community") @@ -339,10 +325,10 @@ def handle_log(data): # If the message is an accept or define type, broadcast it to all # community-connected clients so they can update their display. if data["type"] == "accept": - emit("new_accept", {"uid": scrub_uid(session.uid), "query": data["msg"]["query"], "timestamp": current_unix_time()}, + emit("new_accept", {"uid": session.uid, "query": data["msg"]["query"], "timestamp": current_unix_time()}, broadcast=True, room="community") elif data["type"] == "define": - emit("new_define", {"uid": scrub_uid(session.uid), "defined": data["msg"]["defineAs"], "timestamp": current_unix_time()}, + emit("new_define", {"uid": session.uid, "defined": data["msg"]["defineAs"], "timestamp": current_unix_time()}, broadcast=True, room="community") From a037f9d89cf2c5d37f13e8059d5e6a6a2ba5f5fe Mon Sep 17 00:00:00 2001 From: Sam Ginn Date: Mon, 23 Jan 2017 02:16:53 -0800 Subject: [PATCH 5/5] add scoring function --- community-server/server.py | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/community-server/server.py b/community-server/server.py index e8889d1..8e4c7b2 100755 --- a/community-server/server.py +++ b/community-server/server.py @@ -93,6 +93,8 @@ DATA_FOLDER = "community-server/data/" LOG_FOLDER = os.path.join(DATA_FOLDER, "log/") STRUCTS_FOLDER = os.path.join(DATA_FOLDER, "structs/") +CITATION_FOLDER = "int-output/citation" + # Scoring function parameters GRAVITY = 1.4 # higher the gravity, the faster old structs lose score TIME_INTERVAL = 1800.0 # break off by every 30 minutes @@ -190,6 +192,58 @@ def emit_utterances(): emit("utterances", message) +def h_index(citations): + """https://github.com/kamyu104/LeetCode/blob/master/Python/h-index.py""" + citations.sort(reverse=True) + h = 0 + for x in citations: + if x >= h + 1: + h += 1 + else: + break + return h + + +def compute_citations(dir): + citations = [] + for fname in os.listdir(dir): + if not fname.endswith(".json"): + continue + + path = os.path.join(dir, fname) + + with open(path, 'r') as f: + data = json.load(f) + citations.append(data) + + citation_numbers = [citation["cite"] + (citation["self"] / 10) for citation in citations] + + citation_score = h_index(citation_numbers) + + return (citations, citation_score) + + +def emit_top_builders(): + top_5_builders = [] + for uid in os.listdir(CITATION_FOLDER): + subdir = os.path.join(CITATION_FOLDER, uid) + if not os.path.isdir(subdir): + continue + + (citations, citation_score) = compute_citations(subdir) + + if len(top_5_builders) < 5 or citation_score > top_5_builders[4][0]: + citations = sorted(citations, key=lambda c: c["cite"], reverse=True)[:5] + + struct = (uid, citation_score, citations) + if len(top_5_builders) < 5: + top_5_builders.append(struct) + else: + top_5_builders[4] = struct + + emit("top_builders", {"top_builders": top_5_builders}, broadcast=True, room="community") + + def log(message): """Logs the given message by writing it in the uid's JSON log file.""" uid = session.uid if hasattr(session, 'uid') else "NULL_session" @@ -205,6 +259,15 @@ def log(message): f.write('\n') +@socketio.on('getscore') +def get_score(data): + uid = session.uid + subdir = os.path.join(CITATION_FOLDER, uid) + if (os.path.isdir(subdir)): + (citations, score) = compute_citations(subdir) + emit("score", {"score": score}) + + @socketio.on('join') def on_join(data): """When a user joins the "community" room, emit to them the list of @@ -221,6 +284,9 @@ def on_join(data): # And then we emit the most recent 5 users' utterances per file emit_utterances() + # and also emit the top builders when first joining + emit_top_builders() + @socketio.on('leave') def on_leave(data):