摘要：笔者想下载东西到本地，直接下载速度超慢，一共需要下载1500张图，下载了1天才下载了45张，而且还会报错说请求超时，特绝望，也很无奈，主要原因还是图片储存的服务器在地球的另一边，直接进行传输显然速度很喜感，所以需要一个中转站，先把文件下载到中转服务器，然后再从中转服务器下载下来，笔者使用的是某歌作为文件中转站，某里云作为owncloud的服务器，文件传输如下图所示。
原来的文件传输示意图：
笔者使用的加速路径：
以下操作均在某歌服务器下进行：
笔者将其保存为a.txt，该a.txt文件所在目录认为是当前目录，txt文件如下图所示：
在当前目录下创建data文件夹：
mkdir data
运行以下脚本读取url文件，批量下载文件：
import requests
print ("downloading with requests")
i = 0
with open('./download','r') as f:
#a = f.readline()
for line in f:
url = line[:-1] #去掉换行符
name = url.split('/')[-1]
r = requests.get(url)
print(name)
with open('./data/' + name, "wb") as code:
code.write(r.content)
i = i + 1
print(i)
下载后进行压缩，保存为1.tar.gz：
tar -zcf 1.tar.gz ./data/
先安装owncloud库
pip install pyocclient
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import datetime
import owncloud
class OwncloudUpload:
"""
上传文件到owncloud网盘脚本
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
# 网址
self.url = 'https://www.icdlight.cn/owncloud'
# 用户名
self.username = 'your_username'
# 密码
self.password = 'your_password'
# 上传文件父文件夹
self.dic = 'lw'
# 被上传的文件
self.file = r'./1.tar.gz'
def upload(self):
owncloud_client = self.login()
today_list = self.current_date()
# 判断文件夹是否存在
fold = self.dic
for i in range(3):
fold += '/' + today_list[i]
try:
owncloud_client.file_info(fold)
print('Fold {} is exist.'.format(fold))
except Exception as e:
print('Fold {} is not exist.. HTTP error is: {}'.format(fold, e))
self.creat_fold(owncloud_client, fold)
# 在结尾添加'/'，表示当前为文件夹
fold += '/'
# 上传文件
print('Upload fold is {}.'.format(fold))
self.upload_file(owncloud_client, fold, self.file)
def creat_fold_owncloud(self, datestart="2019-07-18", dateend="2019-10-31"):
owncloud_client = self.login()
date_list = self.create_assist_date(datestart=datestart, dateend=dateend)
for n in date_list:
today_list = n.split('-')
fold = self.dic
for i in range(3):
fold += '/' + today_list[i]
try:
owncloud_client.file_info(fold)
print('Fold {} is exist.'.format(fold))
except Exception as e:
print('Fold {} is not exist.. HTTP error is: {}'.format(fold, e))
self.creat_fold(owncloud_client, fold)
def login(self):
"""
登录网盘
:return: 返回owncloud链接对象
"""
try:
owncloud_client = owncloud.Client(self.url)
owncloud_client.login(self.username, self.password)
print('Login success.')
except Exception as e:
print('Login failed. HTTP error is: {}'.format(e))
return owncloud_client
@staticmethod
def creat_fold(oc, fold):
"""
创建文件夹
:param oc: owncloud_client对象
:param fold: 被创建的文件夹
:return: None
"""
try:
oc.mkdir(fold)
print('Creat {} success.'.format(fold))
except Exception as e:
print('Creat {} failed. HTTP error is: {}'.format(fold, e))
@staticmethod
def upload_file(oc, fold, file):
"""
上传文件
:param oc: owncloud_client对象
:param fold: 目标文件夹
:param file: 被上传的文件
:return: None
"""
try:
oc.put_file(fold, file)
print('Upload {} success.'.format(file))
except Exception as e:
print('Upload {} failed. HTTP error is: {}'.format(file, e))
@staticmethod
def current_date():
"""
获取当前日期
:return: list格式的日期
"""
today = datetime.date.today()
today_list = str(today).split('-')
return today_list
@staticmethod
def create_assist_date(datestart=None, dateend=None):
# 创建日期辅助表
if datestart is None:
datestart = '2016-01-01'
if dateend is None:
dateend = datetime.datetime.now().strftime('%Y-%m-%d')
# 转为日期格式
datestart = datetime.datetime.strptime(datestart, '%Y-%m-%d')
dateend = datetime.datetime.strptime(dateend, '%Y-%m-%d')
date_list = []
date_list.append(datestart.strftime('%Y-%m-%d'))
while datestart < dateend:
# 日期叠加一天
datestart += datetime.timedelta(days=+1)
# 日期转字符串存入列表
date_list.append(datestart.strftime('%Y-%m-%d'))
print(date_list)
return date_list
if __name__ == '__main__':
o = OwncloudUpload()
o.upload()
owncloud有个不好的地方，那就是用脚本上传不支持断点，如果是上传大文件，还是建议拆分了上传，以下是遍历目标文件下的所有文件，并逐一上传
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import datetime
import owncloud
import os
class OwncloudUpload:
"""
上传文件到owncloud网盘脚本
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
# 网址
self.url = 'https://www.icdlight.cn/owncloud'
# 用户名
self.username = 'your_username'
# 密码
self.password = 'your_password'
# 上传文件父文件夹
self.dic = 'lw'
# 被上传的文件
self.file_list = os.listdir('./data')
#self.file = r'./data/33978196618_e30a59e0a8_o.png'
self.file = []
def upload(self):
owncloud_client = self.login()
today_list = self.current_date()
# 判断文件夹是否存在
fold = self.dic
for i in range(3):
fold += '/' + today_list[i]
try:
owncloud_client.file_info(fold)
print('Fold {} is exist.'.format(fold))
except Exception as e:
print('Fold {} is not exist.. HTTP error is: {}'.format(fold, e))
self.creat_fold(owncloud_client, fold)
# 在结尾添加'/'，表示当前为文件夹
fold += '/'
# 上传文件
print('Upload fold is {}.'.format(fold))
#遍历目标文件夹，将一个个文件上传
for file in self.file_list:
file_ = './data/' + file
self.upload_file(owncloud_client, fold, file_)
print(file)
def creat_fold_owncloud(self, datestart="2019-07-18", dateend="2019-10-31"):
owncloud_client = self.login()
date_list = self.create_assist_date(datestart=datestart, dateend=dateend)
for n in date_list:
today_list = n.split('-')
fold = self.dic
for i in range(3):
fold += '/' + today_list[i]
try:
owncloud_client.file_info(fold)
print('Fold {} is exist.'.format(fold))
except Exception as e:
print('Fold {} is not exist.. HTTP error is: {}'.format(fold, e))
self.creat_fold(owncloud_client, fold)
def login(self):
"""
登录网盘
:return: 返回owncloud链接对象
"""
try:
owncloud_client = owncloud.Client(self.url)
owncloud_client.login(self.username, self.password)
print('Login success.')
except Exception as e:
print('Login failed. HTTP error is: {}'.format(e))
return owncloud_client
@staticmethod
def creat_fold(oc, fold):
"""
创建文件夹
:param oc: owncloud_client对象
:param fold: 被创建的文件夹
:return: None
"""
try:
oc.mkdir(fold)
print('Creat {} success.'.format(fold))
except Exception as e:
print('Creat {} failed. HTTP error is: {}'.format(fold, e))
@staticmethod
def upload_file(oc, fold, file):
"""
上传文件
:param oc: owncloud_client对象
:param fold: 目标文件夹
:param file: 被上传的文件
:return: None
"""
try:
oc.put_file(fold, file)
print('Upload {} success.'.format(file))
except Exception as e:
print('Upload {} failed. HTTP error is: {}'.format(file, e))
@staticmethod
def current_date():
"""
获取当前日期
:return: list格式的日期
"""
today = datetime.date.today()
today_list = str(today).split('-')
return today_list
@staticmethod
def create_assist_date(datestart=None, dateend=None):
# 创建日期辅助表
if datestart is None:
datestart = '2016-01-01'
if dateend is None:
dateend = datetime.datetime.now().strftime('%Y-%m-%d')
# 转为日期格式
datestart = datetime.datetime.strptime(datestart, '%Y-%m-%d')
dateend = datetime.datetime.strptime(dateend, '%Y-%m-%d')
date_list = []
date_list.append(datestart.strftime('%Y-%m-%d'))
while datestart < dateend:
# 日期叠加一天
datestart += datetime.timedelta(days=+1)
# 日期转字符串存入列表
date_list.append(datestart.strftime('%Y-%m-%d'))
print(date_list)
return date_list
if __name__ == '__main__':
o = OwncloudUpload()
o.upload()
上传过程中终端显示：
代码参考：https://zhuanlan.zhihu.com/p/73970115