python实现文本去重,不打乱原本顺序-老D代码也是在网上找的,效率挺不错的,特别适合字典文件的去重

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
#coding=utf-8
import sys
def open_txt(): #打开TXT文本写入数组
try:
xxx = file(sys.argv[1], 'r')
for xxx_line in xxx.readlines():
passlist.append(xxx_line)
xxx.close()
except:
return 0

def write_txt(): #打开TXT文本写入数组
try:
yyy = file(sys.argv[2], 'w')
for i in list_passwed:
yyy.write(i)
yyy.close()
except:
return 0

global passlist #声明全局变量
passlist = [] #用户名:anonymous 密码为空
open_txt() #TXT导入数组
#passlist = list(set(passlist)) #python 列表去重
global list_passwed #列表去重,不打乱原来的顺序
list_passwed=[]
for i in passlist:
if i not in list_passwed:
list_passwed.append(i)
write_txt()