From 506fbbf0b1276bf629c36885be59add7aae4087b Mon Sep 17 00:00:00 2001
From: eth3lbert <eth3lbert+dev@gmail.com>
Date: Sun, 14 Apr 2024 04:05:31 +0800
Subject: [PATCH] Prevent storing duplicate records

---
 AutoBuild/nrdlist.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/AutoBuild/nrdlist.py b/AutoBuild/nrdlist.py
index 887b691..a5cecb7 100644
--- a/AutoBuild/nrdlist.py
+++ b/AutoBuild/nrdlist.py
@@ -160,15 +160,14 @@ async def write_files(datalist: List[Dict[str, List[bytes]]]):
                 continue
             combined_data[key].update(value)
 
-    accumulate = b""
+    dataset = set()
     # combined_data is ordered by insertion (sorted date)
     for i, data in enumerate(combined_data.values()):
-        if not accumulate:
-            accumulate = b"\n".join(data)
-        else:
-            accumulate += b"\n" + b"\n".join(data)
+        dataset.update(data)
         # accumulate = "\n".join(sorted(set(accumulate.split("\n"))))
-        base_path.joinpath(f"past-{(i + 1):02d}day.txt").write_bytes(accumulate)
+        base_path.joinpath(f"past-{(i + 1):02d}day.txt").write_bytes(
+            b"\n".join(dataset)
+        )
 
 
 async def main():