-
Notifications
You must be signed in to change notification settings - Fork 1
/
waybacksave.py
97 lines (70 loc) · 3.38 KB
/
waybacksave.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import argparse
import waybackpy
import code
import pprint
from w3lib.url import safe_url_string
# Script version
VERSION = '1.1'
# Options definition
parser = argparse.ArgumentParser(description="version: " + VERSION)
parser.add_argument('-i', '--input-file', help='Input file as list of newline-separated FQDN', required = True)
parser.add_argument('-s', '--output-success', help='Output file to write successfully saved URL')
parser.add_argument('-f', '--output-failed', help='Output file to write failed attempts to save URL')
parser.add_argument('-q', '--quiet', help='Quiet, no output displayed', default = False, action = 'store_true')
def dump_to_file(filename, content):
with open(os.path.abspath(filename), mode='w', encoding='utf-8') as fd_output:
for element in content:
fd_output.write(element + '\n')
return None
def waybackpy_save(options, url, successful_save_attempts, failed_save_attempts):
res = True
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36'
#print("[+] trying to save url:\t\t'%s'" % url)
try:
save_api = waybackpy.WaybackMachineSaveAPI(url=url, user_agent=user_agent, max_tries=5)
save_api.save()
if save_api.status_code == 200:
print("[+] '%s' :\t successfully saved url\n----------" % url) if not(options.quiet) else None
successful_save_attempts.append(url)
except Exception as e:
failed_save_attempts.append(url)
res = False
print("[!] '%s' :\t exception '%s'\n----------" % (url, e.__class__.__name__)) if not(options.quiet) else None
finally:
return res
def submit(options):
successful_save_attempts = []
failed_save_attempts = []
if os.path.isfile(options.input_file):
urls = []
with open(options.input_file, mode='r', encoding='utf-8') as fd_input:
data = fd_input.read().splitlines()
# punydecode
data = list(map(lambda fqdn: safe_url_string(fqdn), data))
if len(data) >= 1:
first_line = data[0]
if not(first_line.startswith(('http://', 'https://'))):
urls = list(map(lambda fqdn: "http://" + fqdn, data)) + list(map(lambda fqdn: "https://" + fqdn, data))
else:
urls = data
if urls:
print("[+] %s urls to save\n" % len(urls)) if not(options.quiet) else None
for url in urls:
waybackpy_save(options, url, successful_save_attempts, failed_save_attempts)
print("\n[!] number of failed save attempts: %s" % len(failed_save_attempts)) if not(options.quiet) else None
pprint.pprint(failed_save_attempts) if failed_save_attempts and not(options.quiet) else None
if options.output_success and successful_save_attempts:
dump_to_file(options.output_success, successful_save_attempts)
if options.output_failed and failed_save_attempts:
dump_to_file(options.output_failed, failed_save_attempts)
return
def main():
global parser
options = parser.parse_args()
submit(options)
return None
if __name__ == "__main__" :
main()