HTTP upload : Check existing remote file, Webgui: remove from timeline

app.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# pilpil-client 0.1
+# pilpil-server 0.1
 # abelliqueux <contact@arthus.net>
 
 import base64
@@ -7,6 +7,7 @@ from flask import Flask, render_template, request, make_response, jsonify
 import gettext
 import http.client
 import os
+import requests
 import ssl
 import subprocess
 from shutil import which
@@ -21,7 +22,7 @@ from waitress import serve
 LOCALE = os.getenv('LANG', 'en_EN')
 _ = gettext.translation('template', localedir='locales', languages=[LOCALE]).gettext
 
-gui_l10n = {"locale" : LOCALE[:2], 
+gui_l10n = {"locale": LOCALE[:2],
             "str_pilpil_title": _("Pilpil-server"),
             "str_filename": _("Media Files"),
             "str_scan": _("Scan"),
@@ -52,10 +53,12 @@ for location in config_locations:
         print(_("Found configuration file in {}").format(os.path.expanduser(location)))
 
 hosts_available, hosts_unavailable = [], []
+
 queue_msgs = [
     _("No items"),
     _("No files queued.")
    ]
+   
 cmd_player = {
     # Map vlc http url parameters to pilpil-server urls
     # See https://github.com/videolan/vlc/blob/1336447566c0190c42a1926464fa1ad2e59adc4f/share/lua/http/requests/README.txt
@@ -85,6 +88,7 @@ cmd_player = {
     # "key"        : "key=",
     "browse": "browse.xml?uri=file://~"
    }
+
 cmd_server = [
     # Map pilpil-client http url parameters to pilpil-server urls
     "blink",
@@ -94,12 +98,24 @@ cmd_server = [
     "sync"
    ]
 
+current_upload = {
+    # status : idle = 0, uploading = 1
+    "status": 0,
+    "progress": -1,
+    "filename": -1,
+    "size": -1,
+    "total_size": -1,
+    "total_count": 0,
+    "transferred_size": 0,
+    "transferred_percent": 0
+    }
+
 # Configuration
 debug = app.config['DEFAULT']['debug']
 pi_user = app.config['DEFAULT']['pi_user']
 media_folder_remote = app.config['DEFAULT']['media_folder_remote']
-media_folder_remote_expanded = media_folder_remote.replace("~/", "/home/{}/".format(pi_user))
+media_folder_remote_expanded = os.path.join(media_folder_remote.replace("~/", "/home/{}/".format(pi_user)), "")
-media_folder_local = os.path.expanduser(app.config['DEFAULT']['media_folder_local'])
+media_folder_local = os.path.join(os.path.expanduser(app.config['DEFAULT']['media_folder_local']), "")
 media_exts = app.config['DEFAULT']['media_exts']
 auth = str(base64.b64encode(str(":" + app.config['DEFAULT']['vlc_auth']).encode('utf-8')), 'utf-8')
 cmd_auth = str(base64.b64encode(str(":" + app.config['DEFAULT']['cmd_auth']).encode('utf-8')), 'utf-8')
@@ -119,8 +135,6 @@ else:
     sslcontext.check_hostname = False
     sslcontext.verify_mode = ssl.CERT_NONE
 
-current_upload = 0
-
 
 # Network/link utilities
 # https://www.metageek.com/training/resources/understanding-rssi/
@@ -166,52 +180,104 @@ def check_hosts(host_list):
     return hosts_up, hosts_down
 
 
-def HTTP_upload(filename, host_local, port, trailing_slash=1):
+def get_upload_candidate_list(host_local, port, media_list):
+    '''
+    Send a JSON request with the media list to the client, which will compare to existing remote files
+    and send back a definitve list of candidates.
+    '''
+    # Send list
+    url = "https://" + host_local + ":" + str(port) + "/upload"
+    http_headers_json_mime = http_headers.copy()
+    http_headers_json_mime["content-type"] = "application/json"
+    post_response = requests.post(url, json=media_list, headers=http_headers_json_mime, verify=CAfile)
+    if debug:
+        print(post_response.text)
+    if post_response.ok:
+        # Get list
+        get_response = requests.get(url, headers=http_headers, verify=CAfile)
+        if get_response.ok:
+            candidates = get_response.json()
+            return candidates
+        else:
+            return []
+    else:
+        if debug:
+            print("Response not ok !")
+        return []
+
+
+def HTTP_upload(file_dict, host_local, port):
     '''
     Build HTTP file upload request and send it.
     '''
     url = "https://" + host_local + ":" + str(port) + "/upload"
-    if not trailing_slash:
+    http_headers_data_mime = http_headers.copy()
-        filename = "/" + filename
+    http_headers_data_mime["content-type"] = ""
-    files = {"file": (filename, open(media_folder_local + filename, "rb"), "multipart/form-data")}
+    files = {"file": (file_dict["filename"], open(media_folder_local + file_dict["filename"], "rb"), "multipart/form-data")}
     if debug:
         print(files)
-    resp = requests.post(url, files=files, headers=http_headers, verify=CAfile)
+    response = requests.post(url, files=files, headers=http_headers, verify=CAfile)
     if debug:
-        print(resp.text)
+        print(response.text)
-    if resp.ok:
+    if response.ok:
         return 1
     else:
         return 0
 
 
+def list_media_files(folder):
+    '''
+    List files in folder which extension is allowed (exists in media_exts).
+    '''
+    if os.path.exists(folder):
+        files = os.listdir(folder)
+        medias = []
+        for fd in files:
+            if len(fd.split('.')) > 1:
+                if fd.split('.')[1] in media_exts:
+                    fd_size = os.stat(folder + fd).st_size
+                    file_dict = {"filename": fd, "size": fd_size}
+                    if debug:
+                        print(str(file_dict))
+                    medias.append(file_dict)
+        return medias
+    else:
+        return []
+
+
 def sync_media_folder(media_folder_local, media_folder_remote, host_local, port, sync_facility=sync_facility):
     '''
     Sync media_folder_local with media_folder_remote using sync_facility
     '''
+    # TODO : first send lift of files to check if they exist and if upload needed
+    global current_upload
     total_size = 0
-    transfer_ok = 0
-    trailing_slash = 1
-    # Check for trailing / and add it if missing
-    if media_folder_local[-1:] != "/":
-        media_folder_local += "/"
-        trailing_slash = 0
-    if media_folder_remote[-1:] != "/":
-        media_folder_remote += "/"
     # Using http_upload
     if sync_facility == "http":
         media_list = list_media_files(media_folder_local)
+        media_list = get_upload_candidate_list(host_local, port, media_list)
+        if debug:
+            print("Media list:" + str(media_list))
+        current_upload["total_count"] = len(media_list)
+        if current_upload["total_count"]:
+            current_upload["status"] = 1
+            media_count = 1
             for media in media_list:
-            current_upload = media.strip("/")
+                total_size += int(media["size"]) / 1024 / 1024
+            current_upload["total_size"] = round(total_size)
+            for media in media_list:
+                current_media_size = int(media["size"]) / 1024 / 1024
+                current_upload["filename"] = media["filename"].strip("/")
+                current_upload["progress"] = media_count
+                current_upload["size"] = round(current_media_size)
                 if debug:
-                print("Uploading " + str(media))
+                    print("Upload candidate : " + str(media))
-            if HTTP_upload(media, host_local, port, trailing_slash):
+                if HTTP_upload(media, host_local, port):
                     if debug:
-                    print("File size: " + str(os.stat(media_folder_local + media).st_size))
+                        print("File size: " + str(round(current_media_size)))
-                total_size += os.stat(media_folder_local + media).st_size
+                    media_count += 1
-                transfer_ok += 1
+                    current_upload["transferred_size"] += round(current_media_size)
-        # ~ return _("{} files uploaded.").format(str(transfer_ok))
+                    current_upload["transferred_percent"] += round((100 / total_size) * current_media_size)
-        # ~ return _("{} files uploaded.").format(str(transfer_ok))
     # Using system cmd
     elif which(sync_facility):
         # Build subprocess arg list accroding to sync_facility
@@ -227,14 +293,14 @@ def sync_media_folder(media_folder_local, media_folder_remote, host_local, port,
             media_list = list_media_files(media_folder_local)
             sync_args = [sync_facility, "-Crp", "-o IdentitiesOnly=yes"]
             for media in media_list:
-                sync_args.append(media_folder_local + media)
+                sync_args.append(media_folder_local + media["filename"])
             sync_args.append(host_local + ":" + media_folder_remote)
         sync_proc = subprocess.run(sync_args, capture_output=True)
         if len(sync_proc.stdout):
             scrape_index = str(sync_proc.stdout).index(scrape_str)+len(scrape_str)
             total_size = str(sync_proc.stdout)[scrape_index:].split(" ")[0][:-1]
     if debug:
-        print("Transferred size: " + str(total_size))
+        print("Transferred size: " + str(round(total_size)))
     return total_size
 
 
@@ -482,7 +548,7 @@ def sync(host):
     # TODO: Add feedback for transfer in GUI
     size = 0
     if host == "status":
-        return str(current_upload)
+        return current_upload
     elif host == "all":
         for hostl in hosts_available:
             size += sync_media_folder(media_folder_local, media_folder_remote_expanded, hostl, cmd_port)

locales/en/LC_MESSAGES/template.pot

@@ -59,6 +59,11 @@ msgstr ""
 msgid "Clear"
 msgstr ""
 
+#: ../app.py:37
+msgid "Delete"
+msgstr ""
+
+
 #: ../app.py:38
 msgid "Sort"
 msgstr ""

locales/fr/LC_MESSAGES/template.pot

@@ -66,6 +66,11 @@ msgstr "Répéter"
 msgid "Clear"
 msgstr "Vider"
 
+#: ../app.py:37
+msgid "Delete"
+msgstr "Supprimer"
+
+
 #: ../app.py:38
 msgid "Sort"
 msgstr "Trier"

requirements.txt

@@ -1,3 +1,4 @@
 flask
 waitress
 toml
+requests

static/script.js

@@ -42,14 +42,14 @@ function sleep(ms) {
 }
 
 
-async function update_VLC_playlist(host) {
+async function update_sort_VLC_playlist(host) {
     // Update host's VLC playlist according to web UI timeline
     let media_count = document.getElementById(`timeline_${host}`).children.length;
     // Reversed loop
     for (let i=media_count, l=0; i>l; i--) {
 	// Find current's timeline element children's 'media_id' value
 	let to_shift = document.getElementById(`timeline_${host}`).children[i-1].children[0].getAttribute("media_id");
-	console.log(to_shift + " : " + document.getElementById(`timeline_${host}`).children[i-1].children[0].innerText);
+	//~ console.log(to_shift + " : " + document.getElementById(`timeline_${host}`).children[i-1].children[0].innerText);
 	// Move 'to_shift' after element with id '1' :
 	// In VLC's playlist XML representation, the playlist node always gets id '1', so moving to that id
 	// really means moving to the the very start of the playlist.
@@ -59,7 +59,16 @@ async function update_VLC_playlist(host) {
     // Un-freeze timeline update flag
     currentUser.freeze_timeline_update = 0;
 }
-
+async function update_delete_VLC_playlist(host, delete_element_id) {
+    let delete_media = document.getElementById(delete_element_id);
+    let delete_media_cont = delete_media.parentElement;
+    let delete_media_id = delete_media.getAttribute("media_id");
+    document.getElementById("timeline_" + host).removeChild(delete_media_cont);
+    send_ajax_cmd("/" + host + "/delete/" + delete_media_id);
+    await sleep(90);
+    adjust_timeline();
+    currentUser.freeze_timeline_update = 0
+}
 
 function find_target_index(element, index) {
     if (element == this) {
@@ -101,6 +110,27 @@ function allow_drop(event) {
 }
 
 
+function drag_over_bin(event) {
+    event.preventDefault();
+    let dropped_id = event.dataTransfer.getData("text");
+    let source_element = document.getElementById(dropped_id);
+    let current_host = source_element.parentElement.parentElement.id.split("_")[1];
+    source_element.style.border = "2px #FFDE00 solid";
+    document.getElementById("delete_" + current_host).style.backgroundColor = "#f00";
+    document.getElementById("delete_" + current_host).style.boxShadow = "0 0 10px #fff;";
+}
+
+
+function drag_leave_bin(event) {
+    event.preventDefault();
+    let dropped_id = event.dataTransfer.getData("text");
+    let source_element = document.getElementById(dropped_id);
+    let current_host = source_element.parentElement.parentElement.id.split("_")[1];
+    source_element.style.border = "none";
+    document.getElementById("delete_" + current_host).style.backgroundColor = "#df7474";
+}
+
+
 function drag(event, source) {
     // Freeze timeline update flag
     currentUser.freeze_timeline_update = 1;
@@ -115,16 +145,21 @@ function drop(event, target_element) {
     // Get dragged element id
     let dropped_id = event.dataTransfer.getData("text");
     let source_element = document.getElementById(dropped_id);
-    let current_host = target_element.parentElement.id.split("_")[1];
+    let current_host = source_element.parentElement.parentElement.id.split("_")[1];
     // Only shift if not dropping on self
     if (source_element.id != target_element.id) {
+	if ( target_element.id.indexOf("delete_") > -1 ) {
+	    update_delete_VLC_playlist(current_host, dropped_id);
+	} else {
 	    let dropTarget = shift_elements(source_element.parentElement, target_element);
-        //~ let dropTarget = shift_elements(currentUser.source_element, target_element);
+	    //~ if (dropTarget) {
-        if (dropTarget) {
 		// Append dropped element to drop target.
 		target_element.appendChild(source_element);
-	    update_VLC_playlist(current_host);
+		update_sort_VLC_playlist(current_host);
+	    //~ }
 	}
+    // Un-freeze timeline update flag
+    //~ currentUser.freeze_timeline_update = 0;
     }
     send_ajax_cmd("/" + current_host + "/list");
 }
@@ -273,9 +308,21 @@ function populate_HTML_table(inner_text, host="all", CSS_class="file_selection")
     document.getElementById("file_sel_" + host).appendChild(tr);
 }
 
+
+function empty_HTML_table(host="all", keep=1) {
+    
+    let HTML_table_element = document.getElementById("file_sel_" + host);
+    let HTML_table_element_length = HTML_table_element.childElementCount;
+    for (let i=keep; i<HTML_table_element_length; HTML_table_element_length--) {
+	HTML_table_element.removeChild(HTML_table_element.children[i]);
+    }
+}
+
+
 function update_remote_filelist(infos_array_element) {
     hosts = Object.keys(infos_array_element);
     for (let j=0, m=hosts.length; j<m; j++) {
+	empty_HTML_table(hosts[j]);
 	let media_item = Object.keys(infos_array_element[hosts[j]]);
 	for (let k=0, n=media_item.length; k<n; k++) {
 	    media_item_meta = infos_array_element[hosts[j]][media_item[k]];
@@ -348,29 +395,31 @@ function parse_result(command, infos_array) {
 	    // Also retrieves loop and repeat status.
 	    infos_array.forEach(update_status);
 	    
-	} else if ( command.indexOf("/list") > -1 ) {
+	} else if (command.indexOf("/list") > -1 ) {
+	    console.log(currentUser.freeze_timeline_update);
 	    // Requests playlist of every instances
 	    if (!currentUser.freeze_timeline_update){
 		infos_array.forEach(update_list);
 	    }
 	    
-	} else if ( command == "/scan") {
+	} else if (command == "/scan") {
 	    // Scan for live hosts
 	    update_host_list(infos_array);
 	    
-	} else if ( command == "/browse_local") {
+	} else if (command == "/browse_local") {
 	    // Display local media files in a table
 	    document.getElementById("filelist").innerHTML = update_local_filelist(infos_array);
 	    
-	} else if ( command == "/all/rssi") {
+	} else if (command == "/all/rssi") {
 	    // RSSI strength indicator
 	    infos_array.forEach(update_rssi_indicator);
 	    
-	} else if ( command == "/all/browse") {
+	//~ } else if ( command == "/all/browse") {
+	} else if (command.indexOf("/browse") > -1) {
 	    // Display remote media files in a table
 	    infos_array.forEach(update_remote_filelist);
 	    
-	} else if ( command == "/sync/status") {
+	} else if (command == "/sync/status") {
 	    // TODO : File sync UI status
 	    currentUser.status_all = infos_array;
 	    
@@ -458,13 +507,15 @@ addEventListener("DOMContentLoaded", function() {
 		request.onload = send_ajax_cmd("/sync/status");
 		
 	    } 
+	    else if ( command.indexOf("/browse") > -1 ) {
+		request.onload = send_ajax_cmd("/all/browse");
+	    }
 
 	    request.open("GET", command, true);
 	    request.send();
 	});
     }
 }, true);
-
 send_ajax_cmd("/scan");
 send_ajax_cmd("/browse_local");
 setInterval(send_ajax_cmd, 500, "/all/status");

static/style.css

@@ -92,6 +92,22 @@ tr:nth-child(2n+1) {background-color: #888;}
     box-shadow: 0 0 18px #91FF7C;
 }
 .btn_txt {display: block;font-size: small;}
+
+.delete_btn {
+    width: 5em;
+    height: 3em;
+    background-color: #df7474;
+    border: 0;
+    float: right;
+    font-size: 2em;
+    line-height: 2em;
+    margin: 0;
+    padding: 0;
+}
+.delete_btn:hover {
+    box-shadow: 0 0 10px #df7474;
+}
+
 /*Right column*/
 .right_col {width: 71%;display: inline-block;}
 /*Left column*/

templates/main.html

@@ -75,6 +75,7 @@
 				<button value="/{{host}}/loop" class="command btn btn-block btn-lg btn-default" role="button">&#x1f501;<span class="btn_txt">{{gui_l10n['str_loop']}}</span></button>
 				<button value="/{{host}}/clear" class="command btn btn-block btn-lg btn-default" role="button">X<span class="btn_txt">{{gui_l10n['str_clear']}}</span></button>
 				<button id="toggle_val_{{host}}}" value="/{{host}}/sort/1/id" class="command btn btn-block btn-lg btn-default" role="button">&#x1f500;<span class="btn_txt">{{gui_l10n['str_sort']}}</span></button>
+				<button id="delete_{{host}}" value="/{{host}}/delete" ondrop="drop(event, this)", ondragover="drag_over_bin(event)" ondragleave="drag_leave_bin(event)" class="delete_btn" role="button">&#x1F5D1;<span class="btn_txt">{{gui_l10n['str_delete']}}</span></button>
 			</div>
 		</div>
 	</div>