pilpil-server/app.py

#!/usr/bin/env python
# pilpil-server 0.1
# abelliqueux <contact@arthus.net>

import base64
from flask import Flask, render_template, request, make_response, jsonify
import gettext
import http.client
import os
import requests
import ssl
import subprocess
from shutil import which
import sys
import toml
from urllib.parse import quote, unquote
import xml.etree.ElementTree as ET
from waitress import serve


# l10n
LOCALE = os.getenv('LANG', 'en_EN')
_ = gettext.translation('template', localedir='locales', languages=[LOCALE]).gettext

gui_l10n = {"locale": LOCALE[:2],
            "str_pilpil_title": _("Pilpil-server"),
            "str_filename": _("Media Files"),
            "str_scan": _("Scan"),
            "str_previous": _("Previous"),
            "str_play": _("Play"),
            "str_pause": _("Pause"),
            "str_stop": _("Stop"),
            "str_next": _("Next"),
            "str_loop": _("Loop"),
            "str_repeat": _("Repeat"),
            "str_clear": _("Clear"),
            "str_sort": _("Sort"),
            "str_sync": _("Sync"),
            "str_poweroff": _("Poweroff"),
            "str_reboot": _("Reboot"),
            # TODO : add to l10n
            "str_poweroff_all": _("Poweroff all"),
            "str_reboot_all": _("Reboot all"),
            "str_blink": _("Blink"),
            "str_link": _("Link"),
            "str_refresh": _("Refresh"),
            }

app = Flask(__name__)

app.config.from_file("defaults.toml", load=toml.load)
config_locations = ["./", "~/.", "~/.config/"]
for location in config_locations:
    # Optional config files, ~ is expanded to $HOME on *nix, %USERPROFILE% on windows
    if app.config.from_file(os.path.expanduser(location + "pilpil-server.toml"), load=toml.load, silent=True):
        print(_("Found configuration file in {}").format(os.path.expanduser(location)))

hosts_available, hosts_unavailable = [], []

queue_msgs = [
    _("No items"),
    _("No files queued.")
   ]
   
cmd_player = {
    # Map vlc http url parameters to pilpil-server urls
    # See https://github.com/videolan/vlc/blob/1336447566c0190c42a1926464fa1ad2e59adc4f/share/lua/http/requests/README.txt
    "play": "pl_play",
    "resume": "pl_forceresume",
    "pause": "pl_forcepause",
    "tpause": "pl_pause",
    "previous": "pl_previous",
    "next": "pl_next",
    "stop": "pl_stop",
    "enqueue": "in_enqueue",
    "add": "in_play",
    "clear": "pl_empty",
    "delete": "pl_delete",
    "loop": "pl_loop",
    "repeat": "pl_repeat",
    "random": "pl_random",
    "move": "pl_move",
    "sort": "pl_sort",
    "seek": "seek",
    "status": "status.xml",
    "list": "playlist.xml",
    # "volume"      : "volume",
    # "ratio"      : "aspectratio",
    # "dir"       : "?dir=<uri>",
    # "command"   : "?command=<cmd>",
    # "key"        : "key=",
    "browse": "browse.xml?uri=file://~"
   }

cmd_server = [
    # Map pilpil-client http url parameters to pilpil-server urls
    "blink",
    "reboot",
    "poweroff",
    "rssi",
    "sync"
   ]

current_upload = {
    # status : idle = 0, uploading = 1, done = -1
    "host": -1, 
    "status": 0,
    "progress": -1,
    "filename": -1,
    "size": -1,
    "total_size": -1,
    "total_count": 0,
    "transferred_size": 0,
    "transferred_percent": 0
    }

stop_upload_flag = 0

# Configuration
debug = app.config['DEFAULT']['debug']
pi_user = app.config['DEFAULT']['pi_user']
media_folder_remote = app.config['DEFAULT']['media_folder_remote']
media_folder_remote_expanded = os.path.join(media_folder_remote.replace("~/", "/home/{}/".format(pi_user)), "")
media_folder_local = os.path.join(os.path.expanduser(app.config['DEFAULT']['media_folder_local']), "")
media_exts = app.config['DEFAULT']['media_exts']
auth = str(base64.b64encode(str(":" + app.config['DEFAULT']['vlc_auth']).encode('utf-8')), 'utf-8')
cmd_auth = str(base64.b64encode(str(":" + app.config['DEFAULT']['cmd_auth']).encode('utf-8')), 'utf-8')
hosts = app.config['DEFAULT']['hosts']
vlc_port = app.config['DEFAULT']['vlc_port']
cmd_port = app.config['DEFAULT']['cmd_port']
useSSL = app.config['DEFAULT']['useSSL']
CAfile = app.config['DEFAULT']['CAfile']
sync_facility = app.config['DEFAULT']['sync_facility']
http_headers = {"Authorization": "Basic " + auth}

# SSl context creation should be out of class
sslcontext = ssl.create_default_context()
if os.path.exists(CAfile):
    sslcontext.load_verify_locations(cafile=CAfile)
else:
    sslcontext.check_hostname = False
    sslcontext.verify_mode = ssl.CERT_NONE


# Network/link utilities

def send_HTTP_request(listed_host, port, time_out=3, request_="/"):
    '''
    Send a http request with http auth
    '''
    if useSSL:
        conn = http.client.HTTPSConnection(listed_host + ":" + str(port), timeout=time_out, context=sslcontext)
    else:
        conn = http.client.HTTPConnection(listed_host + ":" + str(port), timeout=time_out)
    try:
        if debug:
            print(request_ + " - " + str(http_headers))
        conn.request("GET", request_, headers=http_headers)
        resp = conn.getresponse()
        data = resp.read()
        if debug:
            print(_("{} reachable on {}").format(str(listed_host), str(port)))
            print("Data length:" + str(len(data)))
        return data
    except Exception as e:
        if debug:
            print(_("Error on connection to {}: {}: {} ").format(listed_host, str(port), e))
        return 0
    finally:
        conn.close()


def check_hosts(host_list):
    '''
    Check hosts in a host list are up and build then return two lists with up/down hosts.
    '''
    hosts_up, hosts_down = [], []
    hosts_number = str(len(host_list))
    for local_host in host_list:
        if send_HTTP_request(local_host, vlc_port, time_out=1):
            hosts_up.append(local_host)
        else:
            hosts_down.append(local_host)
    if debug:
        print(_("{} of {} hosts found.").format(str(len(hosts_up)), hosts_number))
    return hosts_up, hosts_down


def get_upload_candidate_list(host_local, port, media_list):
    '''
    Send a JSON request with the media list to the client, which will compare to existing remote files
    and send back a definitve list of candidates.
    '''
    # Send list
    url = "https://" + host_local + ":" + str(port) + "/upload"
    http_headers_json_mime = http_headers.copy()
    http_headers_json_mime["content-type"] = "application/json"
    post_response = requests.post(url, json=media_list, headers=http_headers_json_mime, verify=CAfile)
    if debug:
        print(post_response.text)
    if post_response.ok:
        # Get list
        get_response = requests.get(url, headers=http_headers, verify=CAfile)
        if get_response.ok:
            candidates = get_response.json()
            return candidates
        else:
            return []
    else:
        if debug:
            print("Response not ok !")
        return []

def read_in_chunks(file_object, chunk_size=102400):
    while True:
        data = file_object.read(chunk_size)
        if not data:
            break
        yield data

def HTTP_upload(file_dict, host_local, port):
    '''
    Build HTTP file upload request and send it.
    https://stackoverflow.com/questions/43383823/python-flask-chunk-data-upload-not-working/70894476#70894476
    '''
    global current_upload
    http_headers_data_mime = http_headers.copy()
    http_headers_data_mime["content-type"] = ""
    file_path = os.path.join(media_folder_local, file_dict["filename"])
    part_size = int(file_dict["size"] / 10)
    if part_size < 102400:
        part_size = 102400
    url = "https://" + host_local + ":" + str(port) + "/upload/" + file_dict["filename"] + "/" + str(part_size)
    with open(file_path, "rb") as ul_file:
        try:
            for data in read_in_chunks(ul_file, chunk_size=part_size):
                print(len(data))
                # ~ files = {"file": (file_dict["filename"], data, "multipart/form-data")}
                # ~ files = {"file": (file_dict["filename"], "multipart/form-data")}
                # ~ response = requests.post(url, files=files, headers=http_headers, verify=CAfile)
                response = requests.post(url, data=data, headers=http_headers, verify=CAfile)
                # ~ response = requests.post(url, data=data, headers=http_headers, verify=CAfile)
                # ~ if debug:
                    # ~ print(response.text)
                transferred_mb = len(data) / 1024 / 1024
                current_upload["transferred_size"] += round(transferred_mb)
                current_upload["transferred_percent"] += round(100 / current_upload["total_size"] * transferred_mb)
        except Exception as e:
            print(e)
    # ~ files = {"file": (file_dict["filename"], open(media_folder_local + file_dict["filename"], "rb"), "multipart/form-data")}
    # ~ if debug:
        # ~ print(files)
    # ~ response = requests.post(url, files=files, headers=http_headers, verify=CAfile)
    # ~ if debug:
        # ~ print(response.text)
    # ~ if response.ok:
        # ~ return 1
    # ~ else:
        # ~ return 0


def list_media_files(folder):
    '''
    List files in folder which extension is allowed (exists in media_exts).
    '''
    if os.path.exists(folder):
        files = os.listdir(folder)
        medias = []
        for fd in files:
            if len(fd.split('.')) > 1:
                if fd.split('.')[1] in media_exts:
                    fd_size = os.stat(folder + fd).st_size
                    file_dict = {"filename": fd, "size": fd_size}
                    if debug:
                        print(str(file_dict))
                    medias.append(file_dict)
        return medias
    else:
        return []


def reset_current_upload():
    global current_upload
    current_upload = {
    # status : idle = 0, uploading = 1, done = -1
    "host": -1, 
    "status": 0,
    "progress": -1,
    "filename": -1,
    "size": -1,
    "total_size": -1,
    "total_count": 0,
    "transferred_size": 0,
    "transferred_percent": 0
    }


def stop_upload():
    if sync_facility == "http":
        # ~ global stop_upload_flag
        global current_upload
        current_upload["status"] = 0
        # ~ stop_upload_flag = 1
        reset_current_upload()
        return current_upload
    elif sync_facility == "rsync":
        # TODO : This won't work in windows...
        subprocess.run(["pkill", "rsync"])
    else:
        subprocess.run(["pkill", "scp"])
    return _("Interrupting upload...")


def sync_media_folder(media_folder_local, media_folder_remote, host_local, port, sync_facility=sync_facility):
    '''
    Sync media_folder_local with media_folder_remote using sync_facility
    '''
    # TODO : first send lift of files to check if they exist and if upload needed
    global current_upload
    global stop_upload_flag
    total_size = 0
    # Using http_upload
    if sync_facility == "http":
        current_upload["host"] = host_local
        media_list = list_media_files(media_folder_local)
        media_list = get_upload_candidate_list(host_local, port, media_list)
        if debug:
            print("Media list:" + str(media_list))
        current_upload["total_count"] = len(media_list)
        if current_upload["total_count"]:
            current_upload["status"] = 1
            media_count = 1
            for media in media_list:
                total_size += int(media["size"]) / 1024 / 1024
            current_upload["total_size"] = round(total_size)
            for media in media_list:
                # ~ if not stop_upload_flag:
                if current_upload["status"] > 0:
                    current_media_size = int(media["size"]) / 1024 / 1024
                    current_upload["filename"] = media["filename"].strip("/")
                    current_upload["progress"] = media_count
                    current_upload["size"] = round(current_media_size)
                    if debug:
                        print("Upload candidate : " + str(media))
                    if HTTP_upload(media, host_local, port):
                        if debug:
                            print("File size: " + str(round(current_media_size)))
                        media_count += 1
                        # ~ current_upload["transferred_size"] += round(current_media_size)
                        # ~ current_upload["transferred_percent"] += round((100 / total_size) * current_media_size)
                else:
                    # Upload interrupted
                    return _("Upload interrupted")
            # ~ stop_upload_flag = 0
            # ~ reset_current_upload() # host becomes -1
            reset_current_upload()
        current_upload["status"] = -1
    # Using system cmd
    # TODO : fill current_upload with some values from rsync/scp
    elif which(sync_facility):
        # Build subprocess arg list accroding to sync_facility
        # Using Rsync
        if sync_facility == "rsync":
            scrape_str = "total size is "
            sync_args = [sync_facility, "-zharm", "--include='*/'"]
            for media_type in media_exts:
                sync_args.append("--include='*." + media_type + "'")
            sync_args.extend(["--exclude='*'", media_folder_local, host_local + ":" + media_folder_remote])
        # Using scp
        if sync_facility == "scp":
            media_list = list_media_files(media_folder_local)
            sync_args = [sync_facility, "-Crp", "-o IdentitiesOnly=yes"]
            for media in media_list:
                sync_args.append(media_folder_local + media["filename"])
            sync_args.append(host_local + ":" + media_folder_remote)
        sync_proc = subprocess.run(sync_args, capture_output=True)
        if len(sync_proc.stdout):
            scrape_index = str(sync_proc.stdout).index(scrape_str)+len(scrape_str)
            total_size = str(sync_proc.stdout)[scrape_index:].split(" ")[0][:-1]
    if debug:
        print("Transferred size: " + str(round(total_size)))
    return current_upload


def get_meta_data(host, xml_data, request_="status", m3u_=0):
    '''
    Parse XML response from pilpil-client instance and return a dict of metadata according to request type.
    '''
    # Basic metadata
    media_infos = {
      'host': host,
      'status': 0
    }
    if request_ == "list":
        # Return current instance's playlist
        return get_playlist(host, xml_data, m3u_)
    elif request_ == "status":
        # Return current instance's status (currently playing, state, time length, etc.)
        if xml_data.findall("./information/category/"):
            for leaf in xml_data.findall("./information/category/"):
                if leaf.get("name") == "filename":
                    if debug:
                        print(leaf.text)
                    filename = leaf.text
                    break
                else:
                    filename = "N/A"
            cur_length = int(xml_data.find('length').text)
            cur_time = int(xml_data.find('time').text)
            cur_length_fmtd = sec2min(cur_length)
            cur_time_fmtd = sec2min(cur_time)
            cur_id = int(xml_data.find('currentplid').text)
            cur_pos = xml_data.find('position').text
            cur_loop = xml_data.find('loop').text
            cur_repeat = xml_data.find('repeat').text
            media_infos.update({
              'status': 1,
              'file': filename,
              'time': cur_time_fmtd,
              'leng': cur_length_fmtd,
              'pos': cur_pos,
              'loop': cur_loop,
              'repeat': cur_repeat,
              'id': cur_id,
            })
    elif request_ == "rssi":
        # Return current instance's wifi signal quality
        if debug:
            print(xml_data)
        if xml_data.findall("rssi"):
            media_infos.update({
              'status': 1,
              'rssi': xml_data.find('rssi').text
            })
    elif request_ == "browse":
        host_medias = {}
        for media in xml_data:
            media_name = media.attrib["name"]
            if len(media_name.split('.')) > 1:
                if media_name.split('.')[1] in media_exts:
                    host_medias[media_name] = {
                            "name": media_name,
                            "uri": media.attrib["uri"],
                            "size": round(int(media.attrib["size"])/1000000, 2)}
        media_infos = {host: host_medias}
        if debug:
            print(media_infos)
    return media_infos


def get_playlist(host, xml_data, m3u=0):
    playlist = []
    item_list = []
    playlist_duration = 0
    # VLC's playlist node name can change according to the locale on the client, so check for this too.
    if xml_data.find("./node") and (xml_data.find("./node").get('name') == "Playlist" or xml_data.find("./node").get('name') == _("Playlist")):
        playlist = xml_data.findall("./node/leaf")
    content_format = "{};{};{};"
    if m3u:
        m3u_hdr = "#EXTM3U\n"
        m3u_prefix = "#EXTINF:"
        m3u_playlist = m3u_hdr
        # M3U file building
        m3u_format = "{}{}, {}\n{}\n"
        m3u_content = m3u_hdr

    for item in playlist:
        # item info
        if m3u:
            m3u_content += m3u_format.format(m3u_prefix, item.get("duration"), item.get("name"), item.get("uri"))
        item_info = content_format.format(item.get("id"), item.get("name"), sec2min(int(item.get("duration"))))
        # Add cursor to currently playing element
        if "current" in item.keys():
            item_info += item.get("current")
        item_list.append(item_info)
        # Compute playlist length
        playlist_duration += int(item.get("duration"))
    if debug:
        if m3u:
            print(m3u_content)
    playlist_overview = {
                          'host': host,
                          'status': 1,
                          'leng': str(len(playlist)),
                          'duration': sec2min(playlist_duration),
                          'items': item_list
                       }
    if debug:
        print(playlist_overview)
    return playlist_overview


def send_pilpil_command(host, arg0, arg1, arg2):
    '''
    Builds a pilpil request according to args, send it and return parsed result.
    '''
    port_ = vlc_port
    arg0_values = ["play", "delete", "sort", "move"]
    # Build request
    #
    # Default request
    HTTP_request = "/requests/status.xml"
    if arg0 == "list":
        # Get playlist
        HTTP_request = "/requests/playlist.xml"
    elif arg0 == "browse":
        if debug:
            print("Brosing {}".format(media_folder_remote))
        # Browse remote media folder
        media_folder_remote_URL = quote(media_folder_remote_expanded)
        HTTP_request = "/requests/browse.xml?uri=file://{}".format(media_folder_remote_URL)
        if debug:
            print("requesting url {} to {}:{}".format(HTTP_request, host, port_))
    elif arg0 in cmd_server:
        # Switching to cmd server
        HTTP_request = "/" + str(arg0)
        port_ = cmd_port
    elif arg0 != "status":
        # Build request for VLC command
        HTTP_request = HTTP_request + "?command=" + cmd_player[arg0]
        if arg1 != "null":
            # ~ if (arg0 == "play") or (arg0 == "delete") or (arg0 == "sort") or (arg0 == "move"):
            if arg0 in arg0_values:
                # Add 'id' url parameter
                HTTP_request = HTTP_request + "&id=" + arg1
                if (arg0 == "sort") or (arg0 == "move"):
                    # Add 'val' url parameter for "sort"
                    # val possible values: id, title, title nodes first, artist, genre, random, duration, title numeric, album
                    # https://github.com/videolan/vlc/blob/3.0.17.4/modules/lua/libs/playlist.c#L353-L362
                    # For "move", 'val' should be the id of the playlist item we want to move arg1 after.
                    HTTP_request = HTTP_request + "&val=" + escape_str(arg2)
            elif arg0 == "seek":
                HTTP_request = HTTP_request + "&val=" + arg1
            elif (arg0 == "enqueue") or (arg0 == "add"):
                # Add 'input' url parameter
                HTTP_request = HTTP_request + "&input=file://" + os.path.join(quote(media_folder_remote_expanded),arg1)

    # Send request and get data response
    data = send_HTTP_request(host, port_, time_out=3, request_=HTTP_request)
    # ~ if debug:
        # ~ if data:
            # ~ print(str(host) + " - data length:" + str(len(data)) + " : " + str(data))
    if not data:
        print("No data was received.")
        return 0

    # Parse xml data
    xml = ET.fromstring(data)
    # Process parsed data and return dict
    metadata = get_meta_data(host, xml, arg0)
    if debug:
        print("Metadata:" + str(metadata))
    return metadata


# Utilities
def list_local_media_files(folder):
    '''
    List files in folder which extension is allowed (exists in media_exts).
    '''
    if os.path.exists(folder):
        files = os.listdir(folder)
        medias = []
        for fd in files:
            if len(fd.split('.')) > 1:
                if fd.split('.')[1] in media_exts:
                    medias.append(fd)
        return medias
    else:
        return []


# /requests/status.xml?command=in_enqueue&input=file:///home/pi/tst1.mp4
def write_M3U(m3u_content: str, host: str):
    '''
    Write a M3U file named host.m3u from m3u_content
    '''
    filename = host.replace(".", "_") + ".m3u"
    fd = open(filename, "w")
    fd.write(m3u_content)
    fd.close()
    return 1


def escape_str(uri):
    '''
    Replace spaces with %20 for http urls
    '''
    return uri.replace(" ", "%20")


def sec2min(duration):
    '''
    Convert seconds to min:sec format.
    '''
    return ('%02d:%02d' % (duration / 60, duration % 60))


status_message = _("Idle")


@app.route("/")
def main():
    status_message = _("Searching network for live hosts...")
    templateData = {
      'hosts': hosts,
      'status_message': status_message,
      'queue_msgs': queue_msgs,
      'gui_l10n': gui_l10n
     }
    return render_template('main.html', **templateData)


@app.route("/scan")
def scan():
    global hosts_available, hosts_unavailable
    hosts_available, hosts_unavailable = check_hosts(hosts)
    return [hosts_available, hosts_unavailable]


@app.route("/browse_local")
def browse():
    return list_local_media_files(media_folder_local)


@app.route("/sync/<host>/", defaults={"arg0": "null"})
@app.route("/sync/<host>/<arg0>/")
def sync(host, arg0):
    # TODO: move to action() with url /host/sync/...
    global current_upload
    upload_ok = 0
    if arg0 == "status":
        return current_upload
    if arg0 == "stop":
        return stop_upload()
    elif host == "all":
        # This is done in js for now.
        pass
    else:
        reset_current_upload()
        return sync_media_folder(media_folder_local, media_folder_remote_expanded, host, cmd_port)
    return current_upload


@app.route("/<host>/<arg0>/", defaults={"arg1": "null", "arg2": "null"})
@app.route("/<host>/<arg0>/<arg1>/", defaults={"arg2": "null"})
@app.route("/<host>/<arg0>/<arg1>/<arg2>")
def action(host, arg0, arg1, arg2):
    status_message = "Idle"

    if (arg0 not in cmd_player) and (arg0 not in cmd_server):
        status_message = "E:{}".format(_("Wrong command"))
        return status_message

    if host == "all":
        # Send request to all available hosts
        responses = []
        for hostl in hosts_available:
            responses.append(send_pilpil_command(hostl, arg0, arg1, arg2))
        status_message = responses
    elif host not in hosts_available:
        status_message = "<p>{}</p>".format("Host is not reachable")
    else:
        # Send request to specified host
        status_message = [send_pilpil_command(host, arg0, arg1, arg2)]

    if debug:
        print(status_message)

    return status_message


if __name__ == '__main__':
    # ~ app.run()
    HTTP_url_scheme = "http"
    serve(app, host='127.0.0.1', port=5000, url_scheme=HTTP_url_scheme)