yuan/LowTechFilter
1
0
Fork 0
mirror of https://github.com/FutaGuard/LowTechFilter.git synced 2025-06-21 05:21:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Prevent storing duplicate records

Browse Source
This commit is contained in:
eth3lbert 2024-04-14 04:05:31 +08:00
parent a552b790c9
commit 506fbbf0b1
No known key found for this signature in database
1 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
AutoBuild/nrdlist.py
View File

@ -160,15 +160,14 @@ async def write_files(datalist: List[Dict[str, List[bytes]]]):
continue
combined_data[key].update(value)
accumulate = b""
dataset = set()
# combined_data is ordered by insertion (sorted date)
for i, data in enumerate(combined_data.values()):
if not accumulate:
accumulate = b"\n".join(data)
else:
accumulate += b"\n" + b"\n".join(data)
dataset.update(data)
# accumulate = "\n".join(sorted(set(accumulate.split("\n"))))
base_path.joinpath(f"past-{(i + 1):02d}day.txt").write_bytes(accumulate)
base_path.joinpath(f"past-{(i + 1):02d}day.txt").write_bytes(
b"\n".join(dataset)
)
async def main():