20 lines · 433 BytesRaw
data.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/data.py b/data.py
index 92f70f8..5727ed4 100644
--- a/data.py
+++ b/data.py
@@ -1,10 +1,11 @@
+import hashlib
import json
def split(prompts, eval_frac=0.05):
seen = set()
for p in prompts:
- key = p
+ key = hashlib.sha1(normalize(p).encode()).hexdigest()
if key in seen:
continue
seen.add(key)