支持多任务并行,支持传入本地和远程地址,支持下载

pull/1/head
zanllp 2023-03-12 06:10:44 +08:00
parent d57d3863c2
commit 6e07a0920e
18 changed files with 520 additions and 289 deletions

View File

@ -6,8 +6,8 @@
<link rel="icon" href="/favicon.ico">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Vite App</title>
<script type="module" crossorigin src="/baidu_netdisk/fe-static/assets/index-d3a213e0.js"></script>
<link rel="stylesheet" href="/baidu_netdisk/fe-static/assets/index-b702d084.css">
<script type="module" crossorigin src="/baidu_netdisk/fe-static/assets/index-3c97e2e9.js"></script>
<link rel="stylesheet" href="/baidu_netdisk/fe-static/assets/index-cfd42758.css">
</head>
<body>
<div id="zanllp_dev_gradio_fe"></div>

94
scripts/baiduyun_task.py Normal file
View File

@ -0,0 +1,94 @@
import asyncio
import datetime
from typing import List, Dict, Union, Literal
import uuid
import subprocess
from scripts.bin import bin_file_path
class BaiduyunTask:
def __init__(
self,
subprocess: asyncio.subprocess.Process,
type: Literal["upload", "download"],
send_dirs: str,
recv_dir: str,
):
self.subprocess = subprocess
self.id = str(uuid.uuid4())
self.start_time = datetime.datetime.now()
self.running = True
self.logs = []
self.raw_logs = []
self.files_state = {}
self.type = type
self.send_dirs = send_dirs
self.recv_dir = recv_dir
self.n_files = 0
self.n_success_files = 0
self.n_failed_files = 0
def start_time_human_readable(self):
return self.start_time.strftime("%Y-%m-%d %H:%M:%S")
def update_state(self):
self.n_files = 0
self.n_success_files = 0
self.n_failed_files = 0
for key in self.files_state:
status = self.files_state[key]["status"]
self.n_files += 1
if status == "upload-success" or status == "file-skipped":
self.n_success_files += 1
elif status == "upload-failed":
self.n_failed_files += 1
self.running = not isinstance(self.subprocess.returncode, int)
def append_log(self, parsed_log, raw_log):
self.raw_logs.append(raw_log)
self.logs.append(parsed_log)
if isinstance(parsed_log, dict) and "id" in parsed_log:
self.files_state[parsed_log["id"]] = parsed_log
def get_summary(task):
return {
"type": task.type,
"id": task.id,
"running": task.running,
"start_time": task.start_time_human_readable(),
"recv_dir": task.recv_dir,
"send_dirs": task.send_dirs,
"n_files": task.n_files,
"n_failed_files": task.n_failed_files,
"n_success_files": task.n_success_files,
}
@staticmethod
async def create(
type: Literal["upload", "download"], send_dirs: str, recv_dir: str
):
if type not in ["upload", "download"]:
raise Exception("非法参数")
process = await asyncio.create_subprocess_exec(
bin_file_path,
type,
*str(send_dirs).split(","),
recv_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
task = BaiduyunTask(process, type, send_dirs, recv_dir)
task.update_state()
baiduyun_task_cache[task.id] = task
return task
@staticmethod
def get_by_id(id: str):
return baiduyun_task_cache.get(id)
@staticmethod
def get_cache():
return baiduyun_task_cache
baiduyun_task_cache: Dict[str, BaiduyunTask] = {}

View File

@ -9,9 +9,11 @@ import uuid
import asyncio
import subprocess
from modules import script_callbacks, shared
from typing import List, Dict, Union
from typing import List, Dict, Literal, Union
from modules.shared import opts
from scripts.baiduyun_task import BaiduyunTask
import datetime
from pydantic import BaseModel
from scripts.log_parser import parse_log_line
from scripts.bin import (
download_bin_file,
@ -21,6 +23,7 @@ from scripts.bin import (
bin_file_path,
bin_file_name,
)
import functools
# 创建logger对象设置日志级别为DEBUG
@ -241,109 +244,96 @@ def on_ui_settings():
)
class UploadTask:
def __init__(self, subprocess: asyncio.subprocess.Process):
self.subprocess = subprocess
self.id = str(uuid.uuid4())
self.start_time = datetime.datetime.now()
self.running = True
self.logs = []
self.raw_logs = []
self.files_state = {}
self.update_state()
def singleton_async(fn):
@functools.wraps(fn)
async def wrapper(*args, **kwargs):
key = args[0] if len(args) > 0 else None
print(wrapper.busy)
print(key)
if key in wrapper.busy:
raise Exception("Function is busy, please try again later.")
wrapper.busy.append(key)
try:
return await fn(*args, **kwargs)
finally:
wrapper.busy.remove(key)
def start_time_human_readable(self):
return self.start_time.strftime("%Y-%m-%d %H:%M:%S")
def update_state(self):
self.running = not isinstance(self.subprocess.returncode, int)
def append_log(self, parsed_log, raw_log):
self.raw_logs.append(raw_log)
self.logs.append(parsed_log)
if isinstance(parsed_log, dict) and "id" in parsed_log:
self.files_state[parsed_log["id"]] = parsed_log
subprocess_cache: Dict[str, UploadTask] = {}
wrapper.busy = []
return wrapper
def baidu_netdisk_api(_: gr.Blocks, app: FastAPI):
pre = "/baidu_netdisk/"
pre = "/baidu_netdisk"
app.mount(
f"{pre}fe-static",
f"{pre}/fe-static",
StaticFiles(directory=f"{cwd}/vue/dist"),
name="baidu_netdisk-fe-static",
)
@app.get(f"{pre}hello")
@app.get(f"{pre}/hello")
async def greeting():
return "hello"
@app.post(f"{pre}upload")
async def upload():
conf = get_global_conf()
dirs = str(conf["output_dirs"]).split(",")
process = await asyncio.create_subprocess_exec(
bin_file_path,
"upload",
*dirs,
conf["upload_dir"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
task = UploadTask(process)
subprocess_cache[task.id] = task
class BaiduyunUploadDownloadReq(BaseModel):
type: Literal["upload", "download"]
send_dirs: str
recv_dir: str
@app.post(f"{pre}/task")
async def upload(req: BaiduyunUploadDownloadReq):
task = await BaiduyunTask.create(**req.dict())
return {"id": task.id}
@app.get(f"{pre}upload/tasks")
@app.get(f"{pre}/tasks")
async def upload_tasks():
tasks = []
for key in subprocess_cache:
task = subprocess_cache[key]
for key in BaiduyunTask.get_cache():
task = BaiduyunTask.get_by_id(key)
task.update_state()
tasks.append(
{
"id": key,
"running": task.running,
"start_time": task.start_time_human_readable()
}
)
return {"tasks": tasks}
tasks.append(task.get_summary())
return {"tasks": list(reversed(tasks))}
@app.get(pre + "upload/task/{id}/files_state")
@app.get(pre + "/task/{id}/files_state")
async def task_files_stat(id):
p = subprocess_cache.get(id)
p = BaiduyunTask.get_by_id(id)
if not p:
raise HTTPException(status_code=404, detail="找不到该上传任务")
return {
"files_state": p.files_state
}
@app.get(pre + "upload/status/{id}")
return {"files_state": p.files_state}
upload_poll_promise_dict = {}
@app.get(pre + "/task/{id}/tick")
async def upload_poll(id):
p = subprocess_cache.get(id)
if not p:
raise HTTPException(status_code=404, detail="找不到该上传任务")
tasks = []
while True:
try:
line = await asyncio.wait_for(
p.subprocess.stdout.readline(), timeout=0.1
)
line = line.decode()
if not line:
async def get_tick():
task = BaiduyunTask.get_by_id(id)
if not task:
raise HTTPException(status_code=404, detail="找不到该上传任务")
tasks = []
while True:
try:
line = await asyncio.wait_for(
task.subprocess.stdout.readline(), timeout=0.1
)
line = line.decode()
if not line:
break
if line.isspace():
continue
info = parse_log_line(line)
tasks.append({"info": info, "log": line})
task.append_log(info, line)
except asyncio.TimeoutError:
break
if line.isspace():
continue
info = parse_log_line(line)
tasks.append({"info": info, "log": line})
p.append_log(info, line)
except asyncio.TimeoutError:
break
p.update_state()
return {"running": p.running, "tasks": tasks}
task.update_state()
return {"tasks": tasks, "task_summary": task.get_summary()}
res = upload_poll_promise_dict.get(id)
if res:
res = await res
else:
upload_poll_promise_dict[id] = asyncio.create_task(get_tick())
res = await upload_poll_promise_dict[id]
upload_poll_promise_dict.pop(id)
return res
script_callbacks.on_ui_settings(on_ui_settings)

6
vue/components.d.ts vendored
View File

@ -10,6 +10,12 @@ export {}
declare module '@vue/runtime-core' {
export interface GlobalComponents {
AButton: typeof import('ant-design-vue/es')['Button']
AForm: typeof import('ant-design-vue/es')['Form']
AFormItem: typeof import('ant-design-vue/es')['FormItem']
AInput: typeof import('ant-design-vue/es')['Input']
AProgress: typeof import('ant-design-vue/es')['Progress']
ASelect: typeof import('ant-design-vue/es')['Select']
ATextarea: typeof import('ant-design-vue/es')['Textarea']
RouterLink: typeof import('vue-router')['RouterLink']
RouterView: typeof import('vue-router')['RouterView']
}

171
vue/dist/assets/index-3c97e2e9.js vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

1
vue/dist/assets/index-cfd42758.css vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

4
vue/dist/index.html vendored
View File

@ -5,8 +5,8 @@
<link rel="icon" href="/favicon.ico">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Vite App</title>
<script type="module" crossorigin src="/baidu_netdisk/fe-static/assets/index-d3a213e0.js"></script>
<link rel="stylesheet" href="/baidu_netdisk/fe-static/assets/index-b702d084.css">
<script type="module" crossorigin src="/baidu_netdisk/fe-static/assets/index-3c97e2e9.js"></script>
<link rel="stylesheet" href="/baidu_netdisk/fe-static/assets/index-cfd42758.css">
</head>
<body>
<div id="zanllp_dev_gradio_fe"></div>

View File

@ -15,6 +15,8 @@
"dependencies": {
"ant-design-vue": "^3.2.15",
"axios": "^1.3.4",
"pinia": "^2.0.33",
"pinia-plugin-persistedstate": "^3.1.0",
"tsx": "^3.12.3",
"vue": "^3.2.47",
"vue3-ts-util": "^0.8.2"

View File

@ -1,103 +1,17 @@
<!-- eslint-disable no-empty -->
<script setup lang="ts">
import { onMounted, ref, nextTick, reactive, computed } from 'vue'
import { getUploadTaskFilesState, getUploadTasks, getUploadTaskTickStatus, greeting, upload, type UploadTaskFileStatus, type UploadTaskSummary, type UploadTaskTickStatus } from './api'
import { Task } from 'vue3-ts-util'
import { message } from 'ant-design-vue'
const pollTask = ref<ReturnType<typeof createUploadPollTask>>()
const currUploadTaskTickStatusRecord = ref([] as UploadTaskTickStatus[])
const taskLatestInfo = reactive(new Map<string, UploadTaskFileStatus>())
const logListEl = ref<HTMLDivElement>()
const taskSummaryList = ref([] as UploadTaskSummary[])
onMounted(async () => {
await greeting()
const { tasks } = await getUploadTasks()
taskSummaryList.value = tasks
const [runningTask] = tasks.filter(v => v.running)
if (runningTask) {
message.info(`检测到一个运行中的任务,开始还原`)
const { files_state } = await getUploadTaskFilesState(runningTask.id)
Object.entries(files_state).forEach(([k,v]) => taskLatestInfo.set(k,v))
pollTask.value = createUploadPollTask(runningTask.id)
pollTask.value.completedTask.then(() => {
pollTask.value = undefined
})
}
})
const logListScroll2bottom = async () => {
await nextTick()
const el = logListEl.value
if (el) {
el.scrollTop = el.scrollHeight
}
}
const createUploadPollTask = (id: string) => {
const task = Task.run({
action: () => getUploadTaskTickStatus(id),
pollInterval: 500,
validator (r) {
r.tasks.forEach(({ info }) => {
if (info.status === 'start') {
} else if (info.status == 'done') {
} else {
taskLatestInfo.set(info.id, info)
}
})
currUploadTaskTickStatusRecord.value.push(...r.tasks)
logListScroll2bottom()
return !r.running
}
})
return task
}
const onUploadBtnClick = async () => {
currUploadTaskTickStatusRecord.value = []
const { id } = await upload()
pollTask.value = createUploadPollTask(id)
pollTask.value.completedTask.then(() => {
pollTask.value = undefined
})
}
const max = computed(() => taskLatestInfo.size || 100)
const taskLatestInfoArr = computed(() => Array.from(taskLatestInfo))
const done = computed(() => pollTask.value?.task.isFinished)
const uploading = computed(() => pollTask.value?.task.isFinished === false)
const progress = computed(() => {
if (done.value) {
return max.value
}
return taskLatestInfoArr.value.filter(v => v[1].status === 'upload-success' || v[1].status === 'file-skipped' || v[1].status === 'upload-failed').length
})
const progressPercent = computed(() => progress.value * 100 / max.value)
import { SplitView } from 'vue3-ts-util'
import TaskList from './taskList/taskList.vue'
</script>
<template>
<div class="container">
<div class="upload-progress-info" v-if="pollTask">
<progress :max="max" :value="progress" />
<div>
{{ progressPercent.toFixed(2) }} %
</div>
</div>
<div class="action-bar">
<a-button @click="onUploadBtnClick" :disabled="uploading">开始上传</a-button>
</div>
<div>
<div v-for="task in taskSummaryList" :key="task.id">
{{ task.start_time }} 启动的上传任务: {{ task.running ? '进行中' : '已完成' }}
</div>
</div>
<div class="log-list" ref="logListEl" v-if="currUploadTaskTickStatusRecord.length">
<div v-for="msg, idx in currUploadTaskTickStatusRecord" :key="idx">
{{ msg.log }}
</div>
</div>
</div>
<split-view :percent="90">
<template #left>
<task-list>
</task-list>
</template>
</split-view>
</template>
<style scoped>
.action-bar {

View File

@ -7,9 +7,13 @@ export const greeting = async () => {
const resp = await axiosInst.get('hello')
return resp.data as string
}
export const upload = async () => {
const resp = await axiosInst.post('upload')
interface BaiduYunTaskCreateReq {
type: 'upload' | 'download'
send_dirs: string
recv_dir: string
}
export const createBaiduYunTask = async (req: BaiduYunTaskCreateReq) => {
const resp = await axiosInst.post('task', req)
return resp.data as {
id: string
}
@ -68,20 +72,17 @@ export type UploadTaskFileStatus =
export interface UploadTaskTickStatus {
log: string
info:
| UploadTaskDone
| UploadTaskStart
| UploadTaskFileStatus
info: UploadTaskDone | UploadTaskStart | UploadTaskFileStatus
}
/**
*
*/
export const getUploadTaskTickStatus = async (id: string) => {
const resp = await axiosInst.get(`/upload/status/${id}`)
const resp = await axiosInst.get(`/task/${id}/tick`)
return resp.data as {
running: boolean
tasks: UploadTaskTickStatus[]
tasks: UploadTaskTickStatus[],
task_summary: UploadTaskSummary
}
}
@ -89,14 +90,20 @@ export interface UploadTaskSummary {
id: string
running: boolean
start_time: string
send_dirs: string
recv_dir: string
type: 'upload' | 'download'
n_files: number
n_failed_files: number
n_success_files: number
}
/**
*
* @param id
* @param id
*/
export const getUploadTaskFilesState = async (id: string) => {
const resp = await axiosInst.get(`upload/task/${id}/files_state`)
const resp = await axiosInst.get(`/task/${id}/files_state`)
return resp.data as {
files_state: { [x: string]: UploadTaskFileStatus }
}
@ -106,8 +113,8 @@ export const getUploadTaskFilesState = async (id: string) => {
*
*/
export const getUploadTasks = async () => {
const resp = await axiosInst.get('/upload/tasks')
const resp = await axiosInst.get('/tasks')
return resp.data as {
tasks: UploadTaskSummary[]
}
}
}

2
vue/src/icon/index.ts Normal file
View File

@ -0,0 +1,2 @@
export * from '@ant-design/icons-vue'

View File

@ -1,8 +1,12 @@
import { createApp } from 'vue'
import App from './App.vue'
import "antd-vue-volar"
import 'ant-design-vue/es/message/style'
import 'ant-design-vue/es/notification/style'
import 'ant-design-vue/es/modal/style'
createApp(App).mount('#zanllp_dev_gradio_fe')
import { createPinia } from 'pinia'
import piniaPluginPersistedstate from 'pinia-plugin-persistedstate'
const pinia = createPinia()
pinia.use(piniaPluginPersistedstate)
createApp(App).use(pinia).mount('#zanllp_dev_gradio_fe')

View File

@ -0,0 +1,108 @@
<script setup lang="ts">
import { key } from '@/util'
import { onMounted, ref } from 'vue'
import { SearchSelect, type WithId, typedID, Task } from 'vue3-ts-util'
import { PlusOutlined } from '@/icon'
import { createBaiduYunTask, getUploadTasks, getUploadTaskTickStatus, type UploadTaskSummary } from '@/api'
import { message } from 'ant-design-vue'
const tasks = ref<WithId<UploadTaskSummary>[]>([])
const ID = typedID<UploadTaskSummary>(true)
onMounted(async () => {
const resp = await getUploadTasks()
tasks.value = resp.tasks.map(ID)
const runningTasks = tasks.value.filter(v => v.running)
if (runningTasks.length) {
runningTasks.forEach(v => {
createPollTask(v.id).completedTask.then(() => message.success('上传完成'))
})
} else {
addEmptyTask()
}
})
const addEmptyTask = () => {
tasks.value.unshift(
ID({
type: 'upload',
send_dirs: '',
recv_dir: '',
id: '',
running: false,
start_time: '',
n_failed_files: 0,
n_files: 0,
n_success_files: 0
})
)
}
const createNewTask = async (idx: number) => {
const task = tasks.value[idx]
task.running = true
task.n_files = 100
const resp = await createBaiduYunTask(task)
task.id = resp.id
createPollTask(resp.id).completedTask.then(() => message.success('上传完成'))
}
const createPollTask = (id: string) => {
return Task.run({
action: () => getUploadTaskTickStatus(id),
pollInterval: 500,
validator (r) {
const idx = tasks.value.findIndex(v => v.id === id)
tasks.value[idx] = ID(r.task_summary)
return !r.task_summary.running
}
})
}
const getIntPercent = (task: UploadTaskSummary) => parseInt((((task.n_failed_files + task.n_success_files) / task.n_files) * 100).toString())
</script>
<template>
<div class="wrapper">
<a-select style="display: none" />
<a-button @click="addEmptyTask">
<template>
<plus-outlined />
</template>
添加一个任务
</a-button>
<div v-for="task, idx in tasks" :key="key(task)" class="task-form">
<a-form :label-col="{ span: 4 }" :wrapper-col="{ span: 24 }">
<a-form-item label="发送的文件夹">
<a-textarea auto-size :disabled="task.running" v-model:value="task.send_dirs"
placeholder="发送文件的文件夹,多个文件夹使用逗号分隔"></a-textarea>
</a-form-item>
<a-form-item label="接受的文件夹">
<a-input v-model:value="task.recv_dir" :disabled="task.running" placeholder="用于接收的文件夹,可以使用占位符进行动态生成"></a-input>
</a-form-item>
<a-form-item label="任务类型">
<search-select v-model:value="task.type" :disabled="task.running" :options="['upload', 'download']"
:conv="{ value: (v) => v, text: (v) => (v === 'upload' ? '上传' : '下载') }"></search-select>
</a-form-item>
</a-form>
<a-button type="primary" :loading="task.running" :disabled="task.running" @click="createNewTask(idx)"></a-button>
<a-progress v-if="task.running" :stroke-color="{
from: '#108ee9',
to: '#87d068'
}" :percent="getIntPercent(task)" status="active" />
</div>
</div>
</template>
<style scoped lang="scss">
.wrapper {
padding: 8px;
.task-form {
border-radius: 8px;
background: #f8f8f8;
padding: 8px;
margin: 9px;
}
}
</style>

View File

@ -1,3 +1,5 @@
import { idKey, type UniqueId } from 'vue3-ts-util'
export function gradioApp() {
const elems = document.getElementsByTagName('gradio-app')
const gradioShadowRoot = elems.length == 0 ? null : elems[0].shadowRoot
@ -18,4 +20,6 @@ export const asyncCheck = async<T> (getter: () => T, checkSize = 100, timeout =
};
check();
});
}
}
export const key = (obj: UniqueId) => obj[idKey]

View File

@ -28,7 +28,7 @@ export default defineConfig({
server: {
proxy: {
'/baidu_netdisk/': {
target: 'http://127.0.0.1:7861/'
target: 'http://127.0.0.1:7860/'
}
}
}

View File

@ -582,7 +582,7 @@
"@vue/compiler-dom" "3.2.47"
"@vue/shared" "3.2.47"
"@vue/devtools-api@^6.4.5":
"@vue/devtools-api@^6.4.5", "@vue/devtools-api@^6.5.0":
version "6.5.0"
resolved "https://registry.yarnpkg.com/@vue/devtools-api/-/devtools-api-6.5.0.tgz#98b99425edee70b4c992692628fa1ea2c1e57d07"
integrity sha512-o9KfBeaBmCKl10usN4crU53fYtC1r7jJwdGKjPT24t348rHxgfpZ0xL3Xm/gLUYnc0oTp8LAmrxOeLyu6tbk2Q==
@ -2192,6 +2192,19 @@ pify@^4.0.1:
resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231"
integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==
pinia-plugin-persistedstate@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/pinia-plugin-persistedstate/-/pinia-plugin-persistedstate-3.1.0.tgz#eada2b61ecd478fce88e490a685210415cd7a1b4"
integrity sha512-8UN+vYMEPBdgNLwceY08mi5olI0wkYaEb8b6hD6xW7SnBRuPydWHlEhZvUWgNb/ibuf4PvufpvtS+dmhYjJQOw==
pinia@^2.0.33:
version "2.0.33"
resolved "https://registry.yarnpkg.com/pinia/-/pinia-2.0.33.tgz#b70065be697874d5824e9792f59bd5d87ddb5e7d"
integrity sha512-HOj1yVV2itw6rNIrR2f7+MirGNxhORjrULL8GWgRwXsGSvEqIQ+SE0MYt6cwtpegzCda3i+rVTZM+AM7CG+kRg==
dependencies:
"@vue/devtools-api" "^6.5.0"
vue-demi "*"
postcss-selector-parser@^6.0.9:
version "6.0.11"
resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz#2e41dc39b7ad74046e1615185185cd0b17d0c8dc"
@ -2672,6 +2685,11 @@ vite@^4.1.4:
optionalDependencies:
fsevents "~2.3.2"
vue-demi@*:
version "0.13.11"
resolved "https://registry.yarnpkg.com/vue-demi/-/vue-demi-0.13.11.tgz#7d90369bdae8974d87b1973564ad390182410d99"
integrity sha512-IR8HoEEGM65YY3ZJYAjMlKygDQn25D5ajNFNoKh9RSDMQtlzCxtfQjdQgv9jjK+m3377SsJXY8ysq8kLCZL25A==
vue-eslint-parser@^9.0.0, vue-eslint-parser@^9.0.1:
version "9.1.0"
resolved "https://registry.yarnpkg.com/vue-eslint-parser/-/vue-eslint-parser-9.1.0.tgz#0e121d1bb29bd10763c83e3cc583ee03434a9dd5"