JPS-AU-TV/upload.py

# Standard library packages
from pickle import FALSE
import re
import os
import sys
import shutil
import string
import argparse
import html
from urllib.parse import urlparse
import json
import ftplib

# Third-party packages
import requests
from bs4 import BeautifulSoup
from torf import Torrent
from tqdm import tqdm
from langdetect import detect
from pymediainfo import MediaInfo
from pathlib import Path

# JPS-AU files
import jpspy

def asciiart ():
    print("""
     ██╗██████╗ ███████╗       █████╗ ██╗   ██╗   ████████╗██╗   ██╗
     ██║██╔══██╗██╔════╝      ██╔══██╗██║   ██║   ╚══██╔══╝██║   ██║
     ██║██████╔╝███████╗█████╗███████║██║   ██║█████╗██║   ██║   ██║
██   ██║██╔═══╝ ╚════██║╚════╝██╔══██║██║   ██║╚════╝██║   ╚██╗ ██╔╝
╚█████╔╝██║     ███████║      ██║  ██║╚██████╔╝      ██║    ╚████╔╝ 
 ╚════╝ ╚═╝     ╚══════╝      ╚═╝  ╚═╝ ╚═════╝       ╚═╝     ╚═══╝  
                                                                    
""")

# Get arguments using argparse
def getargs():
    """
    Get arguments using argparse
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', help='Initiate upload on input file', nargs='?', required=True)
    parser.add_argument('-d', '--debug', help='Enable debug mode.', action='store_true')
    parser.add_argument("-dry", "--dryrun", help="Dryrun will carry out all actions other than the actual upload to SM.", action="store_true")
    parser.add_argument("-rt", "--releasetype", help='Set the release type. (PV, Music Performance, TV Music, TV Variety, TV-Drama)', nargs='?')
    parser.add_argument("-a", "--artist", help='Set the artist. (Romaji\English). Split multiple with ","', nargs='?')
    parser.add_argument("-oa", "--originalartist", help='Set the artist. (Original Language)', nargs='?')
    parser.add_argument("-ti", "--title", help='Set the title. (Romaji\English)', nargs='?')
    parser.add_argument("-oti", "--originaltitle", help='Set the title. (Original Language)', nargs='?')
    parser.add_argument("-t", "--tags", help="Add additional tags to the upload. At least 2 tags are required", nargs='?')
    parser.add_argument("-y", "--year", help='Set the torrent year (YYYYMMDD or YYYY).', nargs='?')
    parser.add_argument("-fl", "--freeleech", help="Enables freeleech.", action="store_true")
    parser.add_argument("-eti", "--editiontitle", help='Set the edition title', nargs='?')
    parser.add_argument("-ey", "--editionyear", help='Set the torrent edition year (YYYYMMDD or YYYY).', nargs='?')
    parser.add_argument("-ms", "--mediasource", help='Set the media source. (HDTV, Web)', nargs='?')
    parser.add_argument("-im", "--imagepath", help='Set the torrent cover', nargs='?')
    parser.add_argument("-tdes", "--torrentdescription", help='Add a torrent description', nargs='?')
    parser.add_argument("-tgdes", "--torrentgroupdescription", help='Add a torrent group description. This is a required argument.', nargs='?', required=True)
    parser.add_argument("-f", "--formattype", help='Set the media source. (MPEG, MPEG2, AVI, MKV, MP4, h264)', nargs='?')

    return parser.parse_args()

# Acquire the authkey used for torrent files from upload.php
def getauthkey():
    uploadpage = j.retrieveContent("https://jpopsuki.eu/upload.php")
    soup = BeautifulSoup(uploadpage.text, 'html5lib')
    rel2 = str(soup.select('#wrapper #content .thin'))
    # Regex returns multiple matches, could be optimized.
    authkey = re.findall("(?<=value=\")(.*)(?=\")", rel2)[0]
    return authkey
    
    
def copytree(src, dst, symlinks=False, ignore=None):
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            shutil.copytree(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)

# Creates torrent file using torf module.
def createtorrent(authkey, filepath, releasedata):
    t = Torrent(path=filepath,
                trackers=[authkey]) # Torf requires we store authkeys in a list object. This makes it easier to add multiple announce urls.
    # Set torrent to private as standard practice for private trackers
    t.private = True
    t.generate()
    ## Format releasedata to bring a suitable torrent name.
    # The reason we don't just use the directory name is because of an error in POSTING.
    # POSTS do not seem to POST hangul/jp characters alongside files.
    filename = f"{releasedata['title']} [{releasedata['media']}-{releasedata['format']}].torrent"
    filename = filename.replace("/","／").replace(":","：").replace("?","？").replace("\"","")
    try:
        t.write(filename)
        print("_" * 100)
        print("Torrent creation:\n")
        print(f"{filename} has been created.")
    except:
        print("_" * 100)
        print("Torrent creation:\n")
        os.remove(filename)
        print(f"{filename} already exists, existing torrent will be replaced.")
        t.write(filename)
        print(f"{filename} has been created.")

    return filename

def add_to_hangul_dict(hangul , english , category):
    hangul = str(hangul)
    english = str(english)

    categories = ['version','general','artist','genres', 'label', 'distr']
    file = f"json_data/dictionary.json"
    json_file = open(file, 'r', encoding='utf-8', errors='ignore')
    dictionary = json.load(json_file)
    json_file.close()

    new = dict()
    for cats in dictionary:
        #== Create the categories in the new temp file
        new[cats] = dict()

        for key,value in dictionary[cats].items():
            #== List all the old items into the new dict
            new[cats][key] = value

    if hangul in new[category].keys():

        if new[category].get(hangul) is None:

            if english != 'None':
                new[category][hangul] = english

        else:
            #== Only update if English word has been supplied ==#
            if english != 'None':
                new[category][hangul] = english
    else:

        if english == 'None':
            new[category][hangul] = None
        else:
            new[category][hangul] = english

    json_write = open(file, 'w+', encoding='utf-8')
    json_write.write(json.dumps(new, indent=4, ensure_ascii=False))
    json_write.close()

def translate(string, category, result=None, output=None):

    file = "json_data/dictionary.json"
    with open(file, encoding='utf-8', errors='ignore') as f:
        dictionary = json.load(f, strict=False)

    category = str(category)
    string = str(string)
    search = dictionary[category]
    string = string.strip()

    if string == 'Various Artists':
        output = ['Various Artists',None]
    else:
        #== NO NEED TO SEARCH - STRING HAS HANGUL+ENGLISH or HANGUL+HANGUL ==#
        if re.search("\((?P<inside>.*)\)", string):
        #== Complete translation, add to dictionary with both values ==#

            #== Contains parentheses, need to split
            parenthesis = string.split("(")
            pre_parenthesis = parenthesis[0].strip()
            in_parenthesis = parenthesis[1].replace(")","").strip()

            #== Check the order of the parentheses ==#

            if re.search("[^\u0000-\u007F]+",pre_parenthesis) and re.search("[^\u0000-\u007F]+",in_parenthesis):
                #== Both hangul
                first = 'kr'
                second = 'kr'
            else:
                if re.search("[^\u0000-\u007F]+",pre_parenthesis):
                    first = 'kr'
                    second = 'eng'
                else:
                    first = 'eng'
                    second = 'kr'

            if first == 'kr' and second == 'eng':
                #== Hangul first ==#
                hangul = pre_parenthesis
                english = in_parenthesis
                add_to_hangul_dict(hangul,english,category)

            elif first == 'eng' and second == 'kr':
                #== English first ==#
                hangul = in_parenthesis
                english = pre_parenthesis
                add_to_hangul_dict(hangul,english,category)
            elif first == 'kr' and second == 'kr':
                #== Both Hangul ==#
                hangul = pre_parenthesis
                english = None
                add_to_hangul_dict(pre_parenthesis,None,category)
                add_to_hangul_dict(hangul,None,category)
            else:
                #== Both English
                hangul = None
                english = pre_parenthesis

            output = [hangul,english]

        #== No parentheses - HANGUL
        else:

            #== If the input string is a full Hangul word - check dictionary and then add if necessary)
            if re.search("[^\u0000-\u007F]+", string):

                if string in search.keys():
                #== yes
                    if search.get(string) is None:
                        #== If the keyword does not have a translation, add it to the dictionary ==#
                        output = [string,None]
                    else:
                        #== Translation already exists, output the result in a list ==#
                        output = [string,search.get(string)]
                else:
                    output = [string,None]
                    add_to_hangul_dict(string, None, category)

            #== Full English name -- leave it
            else:
                for key,value in search.items():
                    if key == string:
                        output = [value,string]
                        break
                    else:
                        output = [string,string]

    return output

def gatherdata():
    """
    Retrieve data about the upload. Ask for user input if necessary.

    :return: releasedata: dict
    """
    releasedata = {"submit": "true"}
    releasedata["album_desc"]  = torrentgroupdescription

    if torrentdescription:
        releasedata['release_desc'] = torrentdescription

    list_of_types = ["PV", "Music Performance", "TV Music", "TV Variety", "TV-Drama"]
    if releasetype in list_of_types:
        if releasetype == "PV":
            releasedata["type"] = "PV"
        elif releasetype == "TV Music":
            releasedata["type"] = "TV-Music"
        elif releasetype == "TV Variety":
            releasedata["type"] = "TV-Variety"
        elif releasetype == "TV-Drama":
            releasedata["type"] = "TV-Drama"
    else:
        while(True):
            input_lang = input("\n" + "_" * 100 + "\nEnter a number to choose the upload type. \n1=PV\n2=TV-Music\n3=TV-Variety\n4=TV-Drama\n")
            if input_lang == "1":
                releasedata["type"] = "PV"
                break
            elif input_lang == "2":
                releasedata["type"] = "TV-Music"
                break
            elif input_lang == "3":
                releasedata["type"] = "TV-Variety"
                break
            elif input_lang == "4":
                releasedata["type"] = "TV-Drama"
                break
            print("Invalid choice.")

    if artist:
        releasedata['artist'] = artist
    else:
        print(artist)
        input_english_artist = input("\n" + "_" * 100 + "\nEnter the romaji/english ARTIST name.\n")
        releasedata['artist'] = input_english_artist
    
    if originalartist:
        releasedata['artistjp'] = originalartist
    else:
        input_artist = input("\n" + "_" * 100 + "\nEnter the original ARTIST name. Press enter to skip if this torrent has the artist name already in English or already has a artist page.\n")
        releasedata['artistjp'] = input_artist
    
    if title:
        releasedata['title'] = title
    else: 
        input_english_title = input("\n" + "_" * 100 + "\nEnter the romaji/english TITLE:\n")
        releasedata['title'] = input_english_title
    
    if originaltitle:
        releasedata['titlejp'] = originaltitle
    else: 
        input_title = input("\n" + "_" * 100 + "\nEnter the original TITLE. Press enter to skip.\n")
        releasedata['titlejp'] = input_title

    if year:
        releasedata["releasedate"] = year
    else:
        input_year = input("\n" + "_" * 100 + "\nEnter the year as YYYYMMDD or YYYY.\n")
        releasedata["releasedate"] = input_year

    if editiontitle:
        releasedata['remaster_title'] = editiontitle
    else: 
        input_editiontitle = input("\n" + "_" * 100 + "\nEnter the edition TITLE. Press enter to skip.\n")
        if input_editiontitle != "":
            if editionyear:
                releasedata["remaster_year"] = editionyear
            else:
                input_editionyear = input("\n" + "_" * 100 + "\nEnter the edition year as YYYY.\n")
                releasedata["remaster_year"] = input_editionyear
        releasedata['remaster_title'] = input_editiontitle

    if formattype:
        releasedata['format'] = formattype
    else:
        while(True):
            input_format = input("\n" + "_" * 100 + "\nEnter a number to choose the format. \n1=MPEG\n2=MPEG2\n3=AVI\n4=MKV\n5=MP4\n6=h264\n")
            if input_format == "1":
                releasedata["format"] = "MPEG"
                break
            elif input_format == "2":
                releasedata["format"] = "MPEG2"
                break
            elif input_format == "3":
                releasedata["format"] = "AVI"
                break
            elif input_format == "4":
                releasedata["format"] = "MKV"
                break
            elif input_format == "5":
                releasedata["format"] = "MP4"
                break
            elif input_format == "6":
                releasedata["format"] = "h264"
                break
            print("Invalid choice.")

    if mediasource:
        releasedata['media'] = mediasource
    else:
        while(True):
            input_media = input("\n" + "_" * 100 + "\nEnter a number to choose the media source. \n1=HDTV\n2=Web\n")
            if input_media == "1":
                releasedata["media"] = "HDTV"
                break
            elif input_media == "2":
                releasedata["media"] = "Web"
                break
            print("Invalid choice.")

    if tags:
        releasedata["tags"] = tags
    else:
        while(True):
            input_tags = input("\n" + "_" * 100 + "\nEnter the tags. Separate multiple with \",\". Minimum 1 tag required.\n")
            if len(input_tags.split(",")) != 0:
              releasedata["tags"] = input_tags 
              break
            else:
                print("Please enter at least one tag.")

    return releasedata

# MediaInfo.parse doesnt work properly right now. it has duplicate lines
def add_mediainfo_to_releasedata(filename, releasedata):
    """
    Retrieve mediainfo and append it to the releasedata dictionary.
    :return: releasedata: dict
    """
    mediainfosall = ""
    mediainfosall += str(MediaInfo.parse(filename, output="text"))
    replacement = str(Path(filename).parent)
    mediainfosall = mediainfosall.replace(replacement, '')      
    #releasedata["release_desc"] +=  "\n\n" + mediainfosall
    print(mediainfosall)
    return releasedata

# Simple function to split a string up into characters
def split(word):
    return [char for char in word]

def detectlanguage(string):
    ## Language Detect
    # This is a required check as we don't want to enter non-english/romaji characters into the title field.
    characters = split(string)
    language_list = []
    for c in characters:
        try:
            language = detect(c)
            language_list.append(language)
        except:
            langauge = "error"

    if 'ko' or 'ja' in language_list:
        en = False
    else:
        en = True

    return en

def uploadtorrent(torrent, cover, releasedata):

    # POST url.
    uploadurl = "https://jpopsuki.eu/upload.php"

    # Dataset containing all of the information obtained from our FLAC files.
    data = releasedata

    if debug:
        print('_' * 100)
        print('Release Data:\n')
        print(releasedata)

    try:
        postDataFiles = {
            'file_input': open(torrent, 'rb'),
            'userfile': open(cover, 'rb')
        }
    except FileNotFoundError:
	
            print("_" * 100)
            print('File not found!\nPlease confirm file locations and names. Cover image or .torrent file could not be found')
            sys.exit()

    

    # If dryrun argument has not ben passed we will POST the results to JPopSuki.
    if dryrun != True:
        JPSres = j.retrieveContent(uploadurl, "post", data, postDataFiles)
        print('\nUpload POSTED')

    ## TODO Filter through JPSres.text and create error handling based on responses
    #print(JPSres.text)

# Function for transferring the contents of the torrent as well as the torrent.
def ftp_transfer(fileSource, fileDestination, directory, folder_name, watch_folder):

    # Create session
    session = ftplib.FTP(cfg['ftp_prefs']['ftp_server'],cfg['ftp_prefs']['ftp_username'],cfg['ftp_prefs']['ftp_password'])
    # Set session encoding to utf-8 so we can properly handle hangul/other special characters
    session.encoding='utf-8'

    # Successful FTP Login Print
    print("_" * 100)
    print("FTP Login Successful")
    print(f"Server Name: {cfg['ftp_prefs']['ftp_server']}  :  Username: {cfg['ftp_prefs']['ftp_username']}\n")

    if cfg['ftp_prefs']['add_to_downloads_folder']:

        # Create folder based on the directory name of the folder within the torrent.
        try:
            session.mkd(f"{fileDestination}/{folder_name}")
            print(f'Created directory {fileDestination}/{folder_name}')
        except ftplib.error_perm:
            pass

        # Notify user we are beginning the transfer.
        print(f"Beginning transfer...")
        # Set current folder to the users preferred destination
        session.cwd(f"{fileDestination}/{folder_name}")
        # Transfer each file in the chosen directory
        for file in os.listdir(directory):
            with open(f"{directory}/{file}",'rb') as f:
                filesize = os.path.getsize(f"{directory}/{file}")
                ## Transfer file
                # tqdm used for better user feedback.
                with tqdm(unit = 'blocks', unit_scale = True, leave = False, miniters = 1, desc = f'Uploading [{file}]', total = filesize) as tqdm_instance:
                    session.storbinary('STOR ' + file, f, 2048, callback = lambda sent: tqdm_instance.update(len(sent)))
                print(f"{file} | Complete!")
                f.close()

    if cfg['ftp_prefs']['add_to_watch_folder']:
        with open(fileSource,'rb') as t:
            # Set current folder to watch directory
            session.cwd(watch_folder)
            ## Transfer file
            # We avoid tqdm here due to the filesize of torrent files.
            # Most connections will upload these within 1-3s, resulting in near useless progress bars.
            session.storbinary(f"STOR {torrentfile}", t)
            print(f"{torrentfile} | Sent to watch folder!")
            t.close()
    # Quit session when complete.
    session.quit()

def localfileorganization(torrent, directory, watch_folder, downloads_folder):
    # Move torrent directory to downloads_folder
    if cfg['local_prefs']['add_to_downloads_folder']:
        try:
            os.mkdir(os.path.join(downloads_folder, os.path.basename(directory)))
        except FileExistsError:
            pass
        copytree(directory, os.path.join(downloads_folder, os.path.basename(directory)))
        shutil.rmtree(directory)
    if cfg['local_prefs']['add_to_watch_folder']:
        os.rename(torrent, f"{watch_folder}/{torrent}")

def addnewline(text):
    text = re.sub(r"\\n", "\n", text)

def gettorrentgroupdescription(textfile):
    result = ""
    with open(textfile,encoding='utf8') as f:
        lines = f.readlines()
    for line in lines:
        result += line
    return result

def gettorrentdescription(textfile):
    result = ""
    with open(textfile,encoding='utf8') as f:
        lines = f.readlines()
    for line in lines:
        result += line
    return result

if __name__ == "__main__":

    asciiart()
    args = getargs()

    # TODO consider calling args[] directly, we will then not need this line
    dryrun = debug = tags = artist = title = formattype = imagepath = freeleech = None
    originalartist = originaltitle = torrentdescription = torrentgroupdescription = editiontitle = editionyear = year = mediasource = releasetype = None
    inputfile = args.input

    # torrentgroupdescription = args.torrentgroupdescription
    # torrentdescription = args.torrentdescription
    torrentgroupdescription = gettorrentgroupdescription(args.torrentgroupdescription)
    torrentdescription = gettorrentdescription(args.torrentdescription)

    if args.dryrun:
        dryrun = True

    if args.debug:
        debug = True

    if args.freeleech:
        freeleech = True
        
    if args.releasetype:
        releasetype = args.releasetype 

    if args.title:
        title = args.title

    if args.artist:
        artist = args.artist
    
    if args.originalartist:
        originalartist = args.originalartist

    if args.originaltitle:
        originaltitle = args.originaltitle

    if args.editiontitle:
        editiontitle = args.editiontitle
        
    if args.year:
        year = args.year 

    if args.editionyear:
        editionyear = args.editionyear   

    if args.mediasource:
        mediasource = args.mediasource

    if args.formattype:
        format_type = args.formattype

    if args.tags:
        tags = args.tags

    if args.imagepath:
        imagepath = args.imagepath

    # Load login credentials from JSON and use them to create a login session.
    with open(f'json_data/config.json') as f:
        cfg = json.load(f)
    loginData = {'username': cfg['credentials']['username'], 'password': cfg['credentials']['password']}
    loginUrl = "https://jpopsuki.eu/login.php"
    loginTestUrl = "https://jpopsuki.eu"
    successStr = "Latest 5 Torrents"

    # j is an object which can be used to make requests with respect to the loginsession
    j = jpspy.MyLoginSession(loginUrl, loginData, loginTestUrl, successStr, debug=args.debug)
    # Acquire authkey
    authkey = getauthkey()
    # Gather data of the file
    releasedata = gatherdata()
    # releasedata_and_mediainfo = add_mediainfo_to_releasedata(inputfile, releasedata)
    # Folder_name equals the last folder in the path, this is used to rename .torrent files to something relevant.
    # folder_name = os.path.basename(os.path.normpath(directory))
    # Identifying cover.jpg path
    # cover_path = directory + "/" + cfg['local_prefs']['cover_name']

    #= addnewline(releasedata["album_desc"])
    #print(releasedata["album_desc"] )
    # Create torrent file.
    torrentfile = createtorrent(authkey, inputfile, releasedata)

    # Upload torrent to JPopSuki
    uploadtorrent(torrentfile, imagepath, releasedata)

    # Setting variable for watch/download folders
    ftp_watch_folder = cfg['ftp_prefs']['ftp_watch_folder']
    ftp_downloads_folder = cfg['ftp_prefs']['ftp_downloads_folder']
    local_watch_folder = cfg['local_prefs']['local_watch_folder']
    local_downloads_folder = cfg['local_prefs']['local_downloads_folder']


    # if cfg['ftp_prefs']['enable_ftp']:
    #     ftp_transfer(fileSource=torrentfile, fileDestination=ftp_downloads_folder, directory=directory, folder_name=folder_name, watch_folder=ftp_watch_folder)

    # if cfg['local_prefs']['add_to_watch_folder'] or cfg['local_prefs']['add_to_downloads_folder']:
    #     localfileorganization(torrent=torrentfile, directory=directory, watch_folder=local_watch_folder, downloads_folder=local_downloads_folder)

       
    if not dryrun:          
        if cfg['local_prefs']['add_to_watch_folder']:
            os.rename(torrentfile, f"{local_watch_folder}/{torrentfile}")