diff --git a/.gitignore b/.gitignore
index c55de92..cdce0cc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@ logs/
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
+*.info
# Gradle
.idea/**/gradle.xml
@@ -174,6 +175,7 @@ venv.bak/
.mypy_cache/
### VisualStudioCode ###
+.vscode/
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..024875c
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,27 @@
+notifications:
+ email:
+ on_success: never
+ on_failure: always
+language: python
+python: 3.6
+addons:
+ apt:
+ update: true
+env:
+ global:
+ - GH_USER=Twiddly
+ - GH_MAIL=pew@pewpew.moe
+ - REPO=happypandax/plugins
+ - secure: sogo+NT/2/Xr1LNmksFbMSoGCNCgVUcQynhkBiLqSfLrfT9zDXVYl4FGqIntARJSNK1BTvayV/XD0HOtMzbjeARoR91+NYgPScV3PRDu/Bw+X4yM8jjN7GZjz5+a5+co+A8cfuKdrf93CNVV6N02fxzqdC9lhoyp+HD7JxbWKl2+8YmQetrAD4dGf+KaVcJKHr/pgpBND5Tp17jO4vAEbqD+GT7aXDoC+81Onq8UOzNI0/A9s+2IzMlG4Jhfdr52ynpND1plycspByVI3kRqrFPEcWk4x2U5C8OKU6Hf4Zuj3G1d37c8MP6F9F362m6MWa4cfxOeKCmP6uRSVEf9Oxb/w2OnEg40U5nYEpBvziml1STINSgQQlwI4+iO0o1G4Zl7nRsyICZEAhzJRjoPcF5+IfzjGera0xxFkVd+0A9fz8kAyTp53BK5uAxgiaLcIfVj9WsZhSuJPG4kx0abnaNLPJtrDDTdBX9+A0Xk9sXHBCphFVOhobTUspmpCN4zWO9jH8xcPotRyo4D53I7yapSiia89yDxQNLnoNGDVPuX4KTMWO9w2snczEKo9rSJUDJVqdRU1LXbERFCoyCd33Rfm9EZsR06mtkjkpAL7YWOfbjSlyes87OBS3sRW7FqmQPEgSVbovmygOa564Yfo9B48doSKtg09aj0IBJxxXk=
+
+
+script:
+ - python3 build.py
+ - git config --local user.name "$GH_USER"
+ - git config --local user.email "$GH_MAIL"
+ - git commit -am "build"
+ - git push --quiet "https://$GITHUB_TOKEN@github.com/$REPO.git" master > /dev/null 2>&1
+
+branches:
+ only:
+ - master
\ No newline at end of file
diff --git a/Example Plugin/main.py b/Example Plugin/main.py
index 68a8b62..30ea790 100644
--- a/Example Plugin/main.py
+++ b/Example Plugin/main.py
@@ -5,12 +5,15 @@
@hpx.subscribe("init")
def inited():
+ "Called when this plugin is initialised"
pass
@hpx.subscribe("disable")
def disabled():
+ "Called when this plugin has been disiabled"
pass
@hpx.subscribe("remove")
def removed():
+ "Called when this plugin is about to be removed"
pass
\ No newline at end of file
diff --git a/Example Plugin/tests.py b/Example Plugin/test.py
similarity index 100%
rename from Example Plugin/tests.py
rename to Example Plugin/test.py
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..0a04128
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,165 @@
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+ This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+ 0. Additional Definitions.
+
+ As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+ "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+ An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+ A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+ The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+ The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+ 1. Exception to Section 3 of the GNU GPL.
+
+ You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+ 2. Conveying Modified Versions.
+
+ If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+ a) under this License, provided that you make a good faith effort to
+ ensure that, in the event an Application does not supply the
+ function or data, the facility still operates, and performs
+ whatever part of its purpose remains meaningful, or
+
+ b) under the GNU GPL, with none of the additional permissions of
+ this License applicable to that copy.
+
+ 3. Object Code Incorporating Material from Library Header Files.
+
+ The object code form of an Application may incorporate material from
+a header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+ a) Give prominent notice with each copy of the object code that the
+ Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the object code with a copy of the GNU GPL and this license
+ document.
+
+ 4. Combined Works.
+
+ You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+ a) Give prominent notice with each copy of the Combined Work that
+ the Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
+ document.
+
+ c) For a Combined Work that displays copyright notices during
+ execution, include the copyright notice for the Library among
+ these notices, as well as a reference directing the user to the
+ copies of the GNU GPL and this license document.
+
+ d) Do one of the following:
+
+ 0) Convey the Minimal Corresponding Source under the terms of this
+ License, and the Corresponding Application Code in a form
+ suitable for, and under terms that permit, the user to
+ recombine or relink the Application with a modified version of
+ the Linked Version to produce a modified Combined Work, in the
+ manner specified by section 6 of the GNU GPL for conveying
+ Corresponding Source.
+
+ 1) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (a) uses at run time
+ a copy of the Library already present on the user's computer
+ system, and (b) will operate properly with a modified version
+ of the Library that is interface-compatible with the Linked
+ Version.
+
+ e) Provide Installation Information, but only if you would otherwise
+ be required to provide such information under section 6 of the
+ GNU GPL, and only to the extent that such information is
+ necessary to install and execute a modified version of the
+ Combined Work produced by recombining or relinking the
+ Application with a modified version of the Linked Version. (If
+ you use option 4d0, the Installation Information must accompany
+ the Minimal Corresponding Source and Corresponding Application
+ Code. If you use option 4d1, you must provide the Installation
+ Information in the manner specified by section 6 of the GNU GPL
+ for conveying Corresponding Source.)
+
+ 5. Combined Libraries.
+
+ You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+ a) Accompany the combined library with a copy of the same work based
+ on the Library, uncombined with any other library facilities,
+ conveyed under the terms of this License.
+
+ b) Give prominent notice with the combined library that part of it
+ is a work based on the Library, and explaining where to find the
+ accompanying uncombined form of the same work.
+
+ 6. Revised Versions of the GNU Lesser General Public License.
+
+ The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+ If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..d3142f9
--- /dev/null
+++ b/README.md
@@ -0,0 +1,33 @@
+
+#### In this repository resides plugins for HappyPanda X. If you wish to write a plugin for HPX head over to [the docs](https://happypandax.github.io/plugin.html#plugins).
+
+### How to download
+
+I recommend these tools to download a single directory from this repo:
+- https://minhaskamal.github.io/DownGit/ -- *Paste the url to the plugin folder in this repo*
+- https://kinolien.github.io/gitzip/ -- *Paste the url to the plugin folder in this repo*
+- [Firefox Addon](https://addons.mozilla.org/en-US/firefox/addon/gitzip/)
+- [Chrome Extension](https://chrome.google.com/webstore/detail/gitzip-for-github/ffabmkklhbepgcgfonabamgnfafbdlkn)
+
+### How to install
+
+Please see [#Installing plugins](https://happypandax.github.io/usage.html#installing-plugins) in the documentation.
+
+# Be careful about plugins
+
+Read the relevant section [#Be careful about plugins](https://happypandax.github.io/usage.html#be-careful-about-plugins) in the documentation
+
+# Plugins
+
+Name | Version | Description
+--- | --- | ---
+[**Chaika Downloader**](https://github.com/happypandax/plugins/tree/master/plugins/Chaika%20Downloader) | `1.0.0` | *A plugin that enables downloading manga and doujinshi from panda.chaika.moe*
+[**Chaika Metadata**](https://github.com/happypandax/plugins/tree/master/plugins/Chaika%20Metadata) | `1.0.0` | *A plugin that can fetch metadata from Panda.Chaika*
+[**EHentai Downloader**](https://github.com/happypandax/plugins/tree/master/plugins/EHentai%20Downloader) | `1.0.0` | *A plugin that enables downloading manga and doujinshi from E-Hentai & ExHentai*
+[**EHentai Login**](https://github.com/happypandax/plugins/tree/master/plugins/EHentai%20Login) | `1.1.0` | *A plugin that can login to E-Hentai & ExHentai*
+[**EHentai Metadata**](https://github.com/happypandax/plugins/tree/master/plugins/EHentai%20Metadata) | `1.2.1` | *A plugin that can fetch metadata from E-Hentai & ExHentai*
+[**File Metadata**](https://github.com/happypandax/plugins/tree/master/plugins/File%20Metadata) | `2.0.2` | *Extracts and applies metadata from a file accompanying a gallery. Supports files produced from eze, e-hentai-downloader and hdoujin*
+[**NHentai Downloader**](https://github.com/happypandax/plugins/tree/master/plugins/NHentai%20Downloader) | `1.0.1` | *A plugin that enables downloading manga and doujinshi from nhentai.net*
+[**NHentai Metadata**](https://github.com/happypandax/plugins/tree/master/plugins/NHentai%20Metadata) | `1.0.1` | *A plugin that can fetch metadata from nhentai.net*
+
+
diff --git a/build.py b/build.py
new file mode 100644
index 0000000..b67c4bd
--- /dev/null
+++ b/build.py
@@ -0,0 +1,65 @@
+import json
+import glob
+import pathlib
+from urllib.parse import quote
+
+readme = """
+#### In this repository resides plugins for HappyPanda X. If you wish to write a plugin for HPX head over to [the docs](https://happypandax.github.io/plugin.html#plugins).
+
+### How to download
+
+I recommend these tools to download a single directory from this repo:
+- https://minhaskamal.github.io/DownGit/ -- *Paste the url to the plugin folder in this repo*
+- https://kinolien.github.io/gitzip/ -- *Paste the url to the plugin folder in this repo*
+- [Firefox Addon](https://addons.mozilla.org/en-US/firefox/addon/gitzip/)
+- [Chrome Extension](https://chrome.google.com/webstore/detail/gitzip-for-github/ffabmkklhbepgcgfonabamgnfafbdlkn)
+
+### How to install
+
+Please see [#Installing plugins](https://happypandax.github.io/usage.html#installing-plugins) in the documentation.
+
+# Be careful about plugins
+
+Read the relevant section [#Be careful about plugins](https://happypandax.github.io/usage.html#be-careful-about-plugins) in the documentation
+
+# Plugins
+
+{}
+
+"""
+
+plugins_dir = "plugins"
+readme_file = "README.md"
+desc_max_length = 200
+repo_user = "happypandax"
+repo_name = "plugins"
+
+def main():
+ print("Building...")
+ plugin_readme = "Name | Version | Description\n--- | --- | ---\n"
+
+ for p in sorted(glob.glob(f"{plugins_dir}/**/hplugin.json")):
+ with open(p, 'r', encoding="utf-8") as f:
+ d = json.load(f)
+ plugin_dir = pathlib.Path(p).parent
+ dir_name = plugin_dir.name
+ plugin_dir = str(plugin_dir).replace('\\', '/')
+ plugin_desc = d.get("description")
+ plugin_ver = d.get("version")
+
+ gh_url = f"https://github.com/{repo_user}/{repo_name}/tree/master/{quote(plugin_dir)}"
+
+ if plugin_desc and plugin_ver:
+ plugin_desc = plugin_desc.split('\n')[0]
+ if len(plugin_desc) > desc_max_length:
+ plugin_desc = plugin_desc[:desc_max_length] + '…'
+ plugin_readme += f"[**{dir_name}**]({gh_url}) | `{plugin_ver}` | *{plugin_desc}*\n"
+
+ txt = readme.format(plugin_readme)
+
+ with open(readme_file, 'w', encoding="utf-8") as f:
+ f.write(txt)
+ print("Done!")
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
diff --git a/plugins/Chaika Downloader/hplugin.json b/plugins/Chaika Downloader/hplugin.json
new file mode 100644
index 0000000..e78e4f9
--- /dev/null
+++ b/plugins/Chaika Downloader/hplugin.json
@@ -0,0 +1,15 @@
+{
+ "id": "dd86876d-4d4e-438e-bc2e-fbbb18e35742",
+ "shortname": "chaika-downloader",
+ "name": "Chaika Downloader",
+ "version": "1.0.0",
+ "description": "A plugin that enables downloading manga and doujinshi from panda.chaika.moe",
+ "author": "Twiddly",
+ "update_url": "https://github.com/happypandax/plugins/tree/master/plugins/Chaika%20Downloader",
+ "website": "https://github.com/happypandax/plugins/tree/master/plugins/Chaika%20Downloader",
+ "entry": "main.py",
+ "test": "test.py",
+ "require": [
+ "happypandax >= 0.12.0"
+ ]
+}
\ No newline at end of file
diff --git a/plugins/Chaika Downloader/main.py b/plugins/Chaika Downloader/main.py
new file mode 100644
index 0000000..59ccb01
--- /dev/null
+++ b/plugins/Chaika Downloader/main.py
@@ -0,0 +1,149 @@
+# main.py
+import __hpx__ as hpx
+import regex
+
+from bs4 import BeautifulSoup
+
+DownloadRequest = hpx.command.DownloadRequest
+
+log = hpx.get_logger("main")
+
+IDENTIFIER = "chaika"
+HEADERS = {'user-agent':"Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0"}
+DEFAULT_DELAY = 0.5
+
+URLS = {
+ 'ch': 'https://panda.chaika.moe',
+ 'gallery_api': 'https://panda.chaika.moe/jsearch?gallery=',
+}
+
+def website_url_regex_gen(domain, path_regex=None, variable_port=False, variable_tld=False, trailing_slash=True, end=True, trailing_fragment=True):
+ """
+ Generates a regex suitable for a specific domain
+ """
+ rgx = r"^(http\:\/\/|https\:\/\/)?(www\.)?({})".format(domain)
+ if variable_tld:
+ rgx += r"\.[a-z]{2,5}"
+ if variable_port:
+ rgx += r"(:[0-9]{1,5})?"
+ if trailing_slash:
+ rgx += r"\/?"
+ if path_regex:
+ rgx += path_regex
+ if trailing_slash:
+ rgx += r"\/?"
+ if trailing_fragment:
+ rgx += r"(#\S+)?"
+ if end:
+ rgx += "$"
+ return rgx
+
+@hpx.subscribe("init")
+def inited():
+ # set default delay if not set
+ delays = hpx.get_setting("network", "delays", {})
+ delay_url = URLS['ch']
+ if delay_url not in delays:
+ log.info(f"Setting delay on {delay_url} requests to {DEFAULT_DELAY}")
+ delays[delay_url] = DEFAULT_DELAY
+ hpx.update_setting("network", "delays", delays)
+
+@hpx.attach("Download.info")
+def download_info():
+ return hpx.command.DownloadInfo(
+ identifier = IDENTIFIER,
+ name = "Chaika",
+ parser = website_url_regex_gen("panda.chaika.moe", path_regex=r"(gallery|archive)\/[0-9]{3,15}", trailing_slash=True, variable_tld=False, trailing_fragment=True, end=True),
+ sites = ("https://panda.chaika.moe",),
+ description = "Download manga and doujinshi from panda.chaika.moe",
+ )
+
+@hpx.attach("Download.query", trigger=IDENTIFIER)
+def download_query(item):
+ """
+ Called to query for resource URLs that should be downloaded.
+ Note that HPX will handle the actual downloading part.
+ The attached handler should just return all the URLs that should be downloaded in the form of .:class:`DownloadRequest` objects
+
+ should return:
+ a tuple of :class:`DownloadRequest` for all the URL resources that should be downloaded.
+ Note that the download system is recursive, so if the URL resource matches a download handler (the same or a different one),
+ That handler will be called upon with a new :class:`DownloadItem` for that particular URL
+ (though only once, meaning, no handler will be called upon again with the exact same URL during a single session)
+ """
+
+ log.info(f"querying url: {item.url}")
+
+ # prepare request
+ req_props = hpx.command.RequestProperties(
+ headers=HEADERS,
+ )
+
+ # chaika has a simple url system where every download url is in the form of https://panda.chaika.moe/archive/32870/download/
+ # if the url is a gallery url, find and retrieve the archive urls
+
+ url_type, gid = parse_url(item.url)
+
+ download_urls = []
+
+ if url_type == 'gallery':
+ log.info(f"url was a gallery url, retrieving archive urls")
+ req = hpx.command.SingleGETRequest().request(URLS['gallery_api']+str(gid), req_props)
+ if req.ok:
+ log.info("request was successful")
+
+ # get all archive urls
+ a_urls = req.json.get("archives")
+ if a_urls:
+ # we also get to set the name of this download item
+ title = req.json.get('title')
+ if title:
+ item.name = title
+
+ for a in a_urls:
+ download_urls.append(URLS['ch']+a['download'])
+ else:
+ download_urls.append(URLS['ch']+f"/archive/{gid}/download/")
+
+
+ download_requests = []
+
+ if download_urls:
+ log.debug(f"found {len(download_urls)} download urls: {download_urls}")
+ for durl in download_urls:
+ download_requests.append(DownloadRequest(downloaditem=item, url=durl))
+
+ if download_requests:
+ log.info(f"was able to prepare requests for {len(download_requests)} urls")
+ return tuple(download_requests)
+
+@hpx.attach("Download.done", trigger=IDENTIFIER)
+def download_done(result):
+ """
+ Called when downloading of all :class:`DownloadRequest` for a specific :class:`DownloadItem` has finished.
+ The handler should do any post-processing here (archive files, rename files or folders, delete extranous files and etc.).
+ Remember to set the `status` property on the :class:`DownloadResult` object to `False` if the post-processing was a failure.
+ Note that the handler should *not* import the file into HPX (if it's an item), that part will be taken care of by HPX
+
+ should return:
+ the same :class:`DownloadResult` that was provided to the handler, potentially modified on the 'path' or `status` and `reason` properties
+ """
+ # there's nothing special to post-process in the case of chaika downloader, so just return the result as is
+ log.info(f"download of archive was successful for {result.downloaditem.name}")
+ return result
+
+def parse_url(url):
+ "Parses url into a tuple of gallery/archive and id"
+ gallery_id = None
+ stype = "gallery"
+
+ gallery_id = regex.search('([0-9]+)', url)
+ if gallery_id:
+ gallery_id = gallery_id.group()
+ else:
+ log.warning("Error extracting id from url: {}".format(url))
+
+ if 'archive' in url:
+ stype = 'archive'
+
+ return stype, int(gallery_id)
diff --git a/plugins/Chaika Downloader/readme.md b/plugins/Chaika Downloader/readme.md
new file mode 100644
index 0000000..593469b
--- /dev/null
+++ b/plugins/Chaika Downloader/readme.md
@@ -0,0 +1,13 @@
+Chaika Downloader
+----------------------------
+
+> A plugin that enables downloading manga and doujinshi from panda.chaika.moe
+
+## Configuration
+
+There's no available config options for this plugin
+
+# Changelog
+
+- `1.0.0`
+ - first version
\ No newline at end of file
diff --git a/plugins/Chaika Downloader/test.py b/plugins/Chaika Downloader/test.py
new file mode 100644
index 0000000..36a317e
--- /dev/null
+++ b/plugins/Chaika Downloader/test.py
@@ -0,0 +1 @@
+# test.py
\ No newline at end of file
diff --git a/plugins/Chaika Metadata/hplugin.json b/plugins/Chaika Metadata/hplugin.json
new file mode 100644
index 0000000..1b6ab70
--- /dev/null
+++ b/plugins/Chaika Metadata/hplugin.json
@@ -0,0 +1,15 @@
+{
+ "id": "55747a41-789b-43dd-964a-2a6cb1761ff4",
+ "shortname": "chaika-metadata",
+ "name": "Chaika Metadata",
+ "version": "1.0.0",
+ "description": "A plugin that can fetch metadata from Panda.Chaika",
+ "author": "Twiddly",
+ "update_url": "https://github.com/happypandax/plugins/tree/master/plugins/Chaika%20Metadata",
+ "website": "https://github.com/happypandax/plugins/tree/master/plugins/Chaika%20Metadata",
+ "entry": "main.py",
+ "test": "test.py",
+ "require": [
+ "happypandax >= 0.10.0"
+ ]
+}
\ No newline at end of file
diff --git a/plugins/Chaika Metadata/main.py b/plugins/Chaika Metadata/main.py
new file mode 100644
index 0000000..cbb4de3
--- /dev/null
+++ b/plugins/Chaika Metadata/main.py
@@ -0,0 +1,473 @@
+# main.py
+import __hpx__ as hpx
+import regex
+import arrow
+import datetime
+import os
+import urllib
+import html
+
+from bs4 import BeautifulSoup
+from PIL import Image, ImageChops
+
+log = hpx.get_logger("main")
+
+MATCH_URL_PREFIX = r"^(http\:\/\/|https\:\/\/)?(www\.)?" # http:// or https:// + www.
+MATCH_URL_END = r"\/?$"
+
+DEFAULT_DELAY = 1.5
+
+URLS_REGEX = {
+ 'gallery': MATCH_URL_PREFIX + r"(panda\.chaika\.moe\/(archive|gallery)\/[0-9]+)" + MATCH_URL_END,
+}
+
+URLS = {
+ 'ch': 'https://panda.chaika.moe',
+ 'gallery': 'https://panda.chaika.moe/gallery/',
+ 'archive': 'https://panda.chaika.moe/archive/',
+ 'gallery_api': 'https://panda.chaika.moe/jsearch?gallery=',
+ 'archive_api': 'https://panda.chaika.moe/jsearch?archive=',
+ 'hash_api': 'https://panda.chaika.moe/jsearch?sha1=',
+ 'title_search': "https://panda.chaika.moe/galleries/?title={title}&tags=&category=&provider=&uploader=&rating_from=&rating_to=&filesize_from=&filesize_to=&filecount_from=&filecount_to=&sort=posted&asc_desc=desc&apply="
+}
+
+HEADERS = {'user-agent':"Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0"}
+
+PLUGIN_CONFIG = {
+ 'filename_search': False, # use the filename/folder-name for searching instead of gallery title
+ 'remove_namespaces': True, # remove superfluous namespaces like 'artist', 'language' and 'group' because they are handled specially in HPX
+ 'gallery_results_limit': 10, # maximum amount of galleries to return
+ 'blacklist_tags': [], # tags to ignore when updating tags
+ 'add_gallery_url': True, # add ehentai url to gallery
+ 'preferred_language': "english", # preferred gallery langauge (in gallery title) to extract from if multiple galleries were found, set empty string for default
+}
+
+@hpx.subscribe("init")
+def inited():
+ PLUGIN_CONFIG.update(hpx.get_plugin_config())
+
+ # set default delay values if not set
+ delays = hpx.get_setting("network", "delays", {})
+ for u in (URLS['ch'],):
+ if u not in delays:
+ log.info(f"Setting delay on {u} requests to {DEFAULT_DELAY}")
+ delays[u] = DEFAULT_DELAY
+ hpx.update_setting("network", "delays", delays)
+
+@hpx.subscribe('config_update')
+def config_update(cfg):
+ PLUGIN_CONFIG.update(cfg)
+
+@hpx.subscribe("disable")
+def disabled():
+ pass
+
+@hpx.subscribe("remove")
+def removed():
+ pass
+
+@hpx.attach("Metadata.info")
+def metadata_info():
+ return hpx.command.MetadataInfo(
+ identifier = "chaika",
+ name = "Panda.Chaika",
+ parser = URLS_REGEX['gallery'],
+ sites = ("https://panda.chaika.moe",),
+ description = "Fetch metadata from Panda.Chaika",
+ models = (
+ hpx.command.GetDatabaseModel("Gallery"),
+ )
+ )
+
+@hpx.attach("Metadata.query", trigger="chaika")
+def query(itemtuple):
+ """
+ Called to query for candidates to extract metadata from.
+ Note that HPX will handle choosing which candidates to extract data from.
+ The attached handler should just return all the candidates found.
+ """
+ log.info("Querying chaika for metadata")
+ mdata = []
+ for mitem in itemtuple:
+ item = mitem.item
+ url = mitem.url
+ gurls = [] # tuple of (title, url)
+ # url was provided
+ if url:
+ log.info(f"url provided: {url} for {item}")
+ gurls.append((url, url))
+ else: # manually search for id
+ log.info(f"url not provided for {item}")
+ # search with title
+ i_title = ""
+ i_hash = ""
+ if PLUGIN_CONFIG.get("filename_search"):
+ sources = item.get_sources()
+ if sources:
+ # get folder/file name
+ i_title = os.path.split(sources[0])[1]
+ # remove ext
+ i_title = os.path.splitext(i_title)[0]
+ else:
+ if item.titles:
+ i_title = item.titles[0].name # make user choice
+ if i_title:
+ gurls = title_search(i_title)
+
+ # search with hash
+ if not gurls:
+ pass
+
+ log.info(f"found {len(gurls)} urls for item: {item}")
+
+ # list is sorted by date added so we reverse it
+ gurls.reverse()
+
+ log.debug(f"{gurls}")
+ final_gurls = []
+ pref_lang = PLUGIN_CONFIG.get('preferred_language')
+ if pref_lang:
+ for t in gurls:
+ if pref_lang.lower() in t[0].lower():
+ final_gurls.insert(0, t)
+ continue
+ final_gurls.append(t)
+ else:
+ final_gurls = gurls
+
+ for t, u in final_gurls:
+ g_type, g_id = parse_url(u)
+ if g_type and g_id:
+ mdata.append(hpx.command.MetadataData(
+ metadataitem = mitem,
+ title=t,
+ url=u,
+ data={
+ 'type': g_type,
+ 'id': g_id,
+ 'gallery_url': u,
+ }))
+ return tuple(mdata)
+
+@hpx.attach("Metadata.apply", trigger="chaika")
+def apply(datatuple):
+ """
+ Called to fetch and apply metadata to the given data items.
+ Remember to set the `status` property on the :class:`MetadataResult` object to `True` on a successful fetch.
+ """
+ log.info("Applying metadata from chaika")
+ mresult = []
+
+ for mdata in datatuple:
+ applied = False
+ # prepare request
+ req_props = hpx.command.RequestProperties(
+ headers=HEADERS,
+ )
+
+ api_url = URLS['archive_api'] if mdata.data['type'] == 'archive' else URLS['gallery_api']
+ api_url += str(mdata.data['id'])
+
+ r = hpx.command.SingleGETRequest().request(api_url, req_props)
+ if r.ok:
+ response = r.json
+ if response and not 'result' in response:
+ filtered_data = format_metadata(response, mdata.item, apply_url=PLUGIN_CONFIG.get('add_gallery_url', True), gallery_url=mdata.data['gallery_url'])
+ applied = apply_metadata(filtered_data, mdata.item, mdata.options)
+ elif response:
+ log.warning(response)
+ reason = ""
+ if not applied and 'result' in response:
+ reason = response['result']
+ mresult.append(hpx.command.MetadataResult(data=mdata, status=applied, reason=reason))
+ log.info(f"Applied: {applied}")
+ return tuple(mresult)
+
+def title_search(title, session=None, _times=0):
+ "Searches on chaika for galleries with given title, returns a list of (title, matching gallery urls)"
+ search_url = URLS['title_search']
+ log.debug(f"searching with title: {title}")
+ f_url = search_url.format(
+ title=urllib.parse.quote_plus(title)
+ )
+ log.debug(f"final url: {f_url}")
+ r = page_results(f_url, session=session)
+ if not r and not _times:
+ title = regex.sub(r"\(.+?\)|\[.+?\]", "", title)
+ title = " ".join(title.split())
+ r = title_search(title, session, _times=_times+1)
+ return r
+
+def page_results(page_url, limit=None, session=None):
+ "Opens chaika page, parses for results, and then returns list of (title, url)"
+ found_urls = []
+ if limit is None:
+ limit = PLUGIN_CONFIG.get("gallery_results_limit")
+
+ # prepare request
+ req_props = hpx.command.RequestProperties(
+ headers=HEADERS,
+ )
+ if session:
+ req_props.session = session
+ log.debug(f"COOKIES: {session.cookies}")
+ r = hpx.command.SingleGETRequest().request(page_url, req_props)
+ soup = BeautifulSoup(r.text, "html.parser")
+ results = soup.findAll("tr", class_="result-list", limit=limit)
+ results = [r.findAll('td')[1] for r in results]
+ # str(x.a.string)
+ found_urls = [(str(x.a.string), URLS['ch'] + x.a['href']) for x in results] # title, url
+
+ if not found_urls:
+ log.warning(f"No results found on url: {page_url}")
+ log.debug(f"HTML: {r.text}")
+ return found_urls
+
+def parse_url(url):
+ "Parses url into a tuple of gallery/archive and id"
+ gallery_id = None
+ stype = "gallery"
+
+ gallery_id = regex.search('([0-9]+)', url)
+ if gallery_id:
+ gallery_id = gallery_id.group()
+ else:
+ log.warning("Error extracting id from url: {}".format(url))
+
+ if 'archive' in url:
+ stype = 'archive'
+
+ return stype, int(gallery_id)
+
+def capitalize_text(text):
+ """
+ better str.capitalize
+ """
+ return " ".join(x.capitalize() for x in text.strip().split())
+
+def format_metadata(gdata, item, apply_url=False, gallery_url=None):
+ """
+ Formats metadata to look like this for apply_metadata:
+ data = {
+ 'titles': None, # [(title, language),...]
+ 'artists': None, # [(artist, (circle, circle, ..)),...]
+ 'parodies': None, # [parody, ...]
+ 'category': None,
+ 'tags': None, # [tag, tag, tag, ..] or {ns:[tag, tag, tag, ...]}
+ 'pub_date': None, # DateTime object or Arrow object
+ 'language': None,
+ 'urls': None # [url, ...]
+ }
+ """
+ mdata = {}
+
+ mdata['titles'] = []
+
+ parsed_text = hpx.command.ItemTextParser(gdata['title'])
+
+ parsed_title = parsed_text.extract_title()
+ if parsed_title:
+ parsed_title = parsed_title[0]
+ mdata['titles'].append((parsed_title or gdata['title'], 'english'))
+
+ mdata['titles'].append((gdata['title_jpn'], 'japanese'))
+
+
+ mdata['category'] = gdata['category']
+ if gdata['posted']:
+ mdata['pub_date'] = arrow.Arrow.fromtimestamp(gdata['posted'])
+
+ lang = "japanese" # default language
+
+ artists = set()
+ circles = set()
+ parodies = set()
+
+ parsed_artists = parsed_text.extract_artist()
+ parsed_circles = parsed_text.extract_circle()
+
+ extranous_namespaces = ("artist", "parody", "group", "language")
+ mdata['tags'] = {}
+
+ for nstag in gdata['tags']:
+ onstag = nstag
+ nstag = nstag.replace('_', ' ')
+ blacklist_tags = PLUGIN_CONFIG.get("blacklist_tags")
+ if blacklist_tags and (nstag in blacklist_tags or onstag in blacklist_tags):
+ continue
+
+ ns = None
+ if ':' in nstag:
+ ns, t = nstag.split(':', 1)
+ else:
+ t = nstag
+
+ if ns == 'language' and t != 'translated':
+ lang = t
+ elif ns == "artist":
+ for a in artists: # the artist extracted from the title likely has better capitalization, so choose that instead
+ if a.lower() == t.lower():
+ artists.add(a)
+ break
+ else:
+ artists.add(t)
+ elif ns == "group":
+ for c in circles: # the circle extracted from the title likely has better capitalization, so choose that instead
+ if c.lower() == t.lower():
+ circles.add(c)
+ break
+ else:
+ circles.add(t)
+ elif ns == "parody":
+ parodies.add(t)
+
+ if not (PLUGIN_CONFIG.get("remove_namespaces") and ns in extranous_namespaces):
+ mdata['tags'].setdefault(ns, []).append(t)
+ else:
+ log.debug(f"removing namespace {ns}")
+
+ log.debug(f"tags: {mdata['tags']}")
+
+ mdata['language'] = lang
+
+ if parodies:
+ mdata['parodies'] = parodies
+
+ if artists:
+ a_circles = []
+ for a in artists:
+ a_circles.append((a, tuple(circles))) # assign circles to each artist
+ mdata['artists'] = a_circles
+
+ if apply_url:
+ if gdata.get('gallery', False):
+ mdata['urls'] = [URLS['gallery']+f"{gdata['gallery']}/"]
+ elif gallery_url:
+ mdata['urls'] = [gallery_url]
+
+ return mdata
+
+GalleryData = hpx.command.GalleryData
+LanguageData = hpx.command.LanguageData
+TitleData = hpx.command.TitleData
+ArtistData = hpx.command.ArtistData
+ArtistNameData = hpx.command.ArtistNameData
+ParodyData = hpx.command.ParodyData
+ParodyNameData = hpx.command.ParodyNameData
+CircleData = hpx.command.CircleData
+CategoryData = hpx.command.CategoryData
+UrlData = hpx.command.UrlData
+NamespaceTagData= hpx.command.NamespaceTagData
+TagData= hpx.command.TagData
+NamespaceData = hpx.command.NamespaceData
+
+def apply_metadata(data, gallery, options):
+ """
+ data = {
+ 'titles': None, # [(title, language),...]
+ 'artists': None, # [(artist, (circle, circle, ..)),...]
+ 'parodies': None, # [parody, ...]
+ 'category': None,
+ 'tags': None, # [tag, tag, tag, ..] or {ns:[tag, tag, tag, ...]}
+ 'pub_date': None, # DateTime object or Arrow object
+ 'language': None,
+ 'urls': None # [url, ...]
+ }
+ """
+
+ log.debug(f"data: {data}")
+
+ gdata = GalleryData()
+
+ if isinstance(data.get('titles'), (list, tuple, set)):
+ gtitles = []
+ for t, l in data['titles']:
+ gtitle = None
+ if t:
+ t = html.unescape(t)
+ gtitle = TitleData(name=t)
+ if t and l:
+ gtitle.language = LanguageData(name=l)
+ if gtitle:
+ gtitles.append(gtitle)
+
+ if gtitles:
+ gdata.titles = gtitles
+ log.debug("applied titles")
+
+ if isinstance(data.get('artists'), (list, tuple, set)):
+ gartists = []
+ for a, c in data['artists']:
+ if a:
+ gartist = ArtistData(names=[ArtistNameData(name=capitalize_text(a))])
+ gartists.append(gartist)
+
+ if c:
+ gcircles = []
+ for circlename in [x for x in c if x]:
+ gcircles.append(CircleData(name=capitalize_text(circlename)))
+ gartist.circles = gcircles
+
+ if gartists:
+ gdata.artists = gartists
+ log.debug("applied artists")
+
+ if isinstance(data.get('parodies'), (list, tuple, set)):
+ gparodies = []
+ for p in data['parodies']:
+ if p:
+ gparody = ParodyData(names=[ParodyNameData(name=capitalize_text(p))])
+ gparodies.append(gparody)
+
+ if gparodies:
+ gdata.parodies = gparodies
+ log.debug("applied parodies")
+
+ if data.get('category'):
+ gdata.category = CategoryData(name=data['category'])
+ log.debug("applied category")
+
+ if data.get('language'):
+ gdata.language = LanguageData(name=data['language'])
+ log.debug("applied language")
+
+ if isinstance(data.get('tags'), (dict, list)):
+ if isinstance(data['tags'], list):
+ data['tags'] = {None: data['tags']}
+ gnstags = []
+ for ns, tags in data['tags'].items():
+ if ns is not None:
+ ns = ns.strip()
+ if ns and ns.lower() == 'misc':
+ ns = None
+ for t in tags:
+ t = t.strip()
+ if t:
+ kw = {'tag': TagData(name=t)}
+ if ns:
+ kw['namespace'] = NamespaceData(name=ns)
+ gnstags.append(NamespaceTagData(**kw))
+
+ if gnstags:
+ gdata.tags = gnstags
+ log.debug("applied tags")
+
+ if isinstance(data.get('pub_date'), (datetime.datetime, arrow.Arrow)):
+ pub_date = data['pub_date']
+ gdata.pub_date = pub_date
+ log.debug("applied pub_date")
+
+ if isinstance(data.get('urls'), (list, tuple)):
+ gurls = []
+ for u in data['urls']:
+ if u:
+ gurls.append(UrlData(name=u))
+ if gurls:
+ gdata.urls = gurls
+ log.debug("applied urls")
+
+ applied = hpx.command.UpdateItemData(gallery, gdata, options=options)
+
+ log.debug(f"applied: {applied}")
+
+ return applied
\ No newline at end of file
diff --git a/plugins/Chaika Metadata/readme.md b/plugins/Chaika Metadata/readme.md
new file mode 100644
index 0000000..28461ee
--- /dev/null
+++ b/plugins/Chaika Metadata/readme.md
@@ -0,0 +1,41 @@
+Chaika Metadata
+----------------------------
+
+> This plugin fetches metadata from Panda.Chaika
+
+## Configuration
+
+Configure this plugin by adding `chaika-metadata` to the `plugin.config` namespace in your `config.yaml`:
+```yaml
+plugin:
+ config:
+ chaika-metadata:
+ option1: True
+ option2:
+ - item 1
+ - item 2
+```
+
+#### Available options
+
+Name | Default | Description
+--- | --- | ---
+`filename_search` | `false` | use the filename/folder-name for searching instead of gallery title
+`remove_namespaces` | `true` | remove superfluous namespaces like 'artist', 'language' and 'group' because they are handled specially in HPX
+`gallery_results_limit` | `10` | maximum amount of galleries to return
+`blacklist_tags` | `[]` | tags to ignore when updating tags, a list of `namespace:tag` or `tag` strings
+`add_gallery_url` | `true` | add chaika url to gallery
+`preferred_language` | `english` | preferred gallery langauge (in gallery title) to extract from if multiple galleries were found, set empty string for default
+
+
+## Things yet to be implemented
+
+- File similarity search
+
+# Changelog
+
+- `1.0.0`
+ - Updated to reflect new changes in HPX v0.10.0
+
+- `0.1.0b`
+ - first version
\ No newline at end of file
diff --git a/plugins/Chaika Metadata/test.py b/plugins/Chaika Metadata/test.py
new file mode 100644
index 0000000..36a317e
--- /dev/null
+++ b/plugins/Chaika Metadata/test.py
@@ -0,0 +1 @@
+# test.py
\ No newline at end of file
diff --git a/plugins/EHentai Downloader/hplugin.json b/plugins/EHentai Downloader/hplugin.json
new file mode 100644
index 0000000..21eb484
--- /dev/null
+++ b/plugins/EHentai Downloader/hplugin.json
@@ -0,0 +1,15 @@
+{
+ "id": "efaec768-760c-49d7-8d45-fb70b7db45e5",
+ "shortname": "ehentai-downloader",
+ "name": "EHentai Downloader",
+ "version": "1.0.0",
+ "description": "A plugin that enables downloading manga and doujinshi from E-Hentai & ExHentai",
+ "author": "Twiddly",
+ "update_url": "https://github.com/happypandax/plugins/tree/master/plugins/EHentai%20Downloader",
+ "website": "https://github.com/happypandax/plugins/tree/master/plugins/EHentai%20Downloader",
+ "entry": "main.py",
+ "test": "test.py",
+ "require": [
+ "happypandax >= 0.12.0"
+ ]
+}
\ No newline at end of file
diff --git a/plugins/EHentai Downloader/main.py b/plugins/EHentai Downloader/main.py
new file mode 100644
index 0000000..bfaaa06
--- /dev/null
+++ b/plugins/EHentai Downloader/main.py
@@ -0,0 +1,251 @@
+# main.py
+import __hpx__ as hpx
+import regex
+import json
+from bs4 import BeautifulSoup
+
+DownloadRequest = hpx.command.DownloadRequest
+
+log = hpx.get_logger("main")
+
+EH_IDENTIFIER = "ehentai"
+EX_IDENTIFIER = "exhentai"
+
+HEADERS = {'user-agent':"Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0"}
+DEFAULT_DELAY = 5
+
+URLS = {
+ 'eh': 'https://e-hentai.org',
+ 'ex': 'https://exhentai.org',
+ 'e_api': 'https://api.e-hentai.org/api.php',
+ 'ex_api': 'https://exhentai.org/api.php',
+ 'e_archiver': 'https://e-hentai.org/archiver.php?gid={gallery_id}&token={gallery_token}&or={archiver_key}',
+ 'ex_archiver': 'https://exhentai.org/archiver.php?gid={gallery_id}&token={gallery_token}&or={archiver_key}',
+}
+
+
+def website_url_regex_gen(domain, path_regex=None, variable_port=False, variable_tld=False, trailing_slash=True, end=True, trailing_fragment=True):
+ """
+ Generates a regex suitable for a specific domain
+ """
+ rgx = r"^(http\:\/\/|https\:\/\/)?(www\.)?({})".format(domain)
+ if variable_tld:
+ rgx += r"\.[a-z]{2,5}"
+ if variable_port:
+ rgx += r"(:[0-9]{1,5})?"
+ if trailing_slash:
+ rgx += r"\/?"
+ if path_regex:
+ rgx += path_regex
+ if trailing_slash:
+ rgx += r"\/?"
+ if trailing_fragment:
+ rgx += r"(#\S+)?"
+ if end:
+ rgx += "$"
+ return rgx
+
+@hpx.subscribe("init")
+def inited():
+ # set default delay values if not set
+ delays = hpx.get_setting("network", "delays", {})
+ for u in (URLS['ex'], URLS['eh'], "https://api.e-hentai.org", URLS['ex_api']):
+ if u not in delays:
+ log.info(f"Setting delay on {u} requests to {DEFAULT_DELAY}")
+ delays[u] = DEFAULT_DELAY
+ hpx.update_setting("network", "delays", delays)
+
+@hpx.attach("Download.info")
+def eh_download_info():
+ return hpx.command.DownloadInfo(
+ identifier = EH_IDENTIFIER,
+ name = "E-Hentai",
+ parser = website_url_regex_gen("e-hentai.org", path_regex=r"g\/[0-9]{3,10}\/[0-9a-zA-Z]{3,15}", trailing_slash=True, variable_tld=False, trailing_fragment=True, end=True),
+ sites = ("https://e-hentai.org",),
+ description = "Download manga and doujinshi from e-hentai.org",
+ )
+
+@hpx.attach("Download.info")
+def ex_download_info():
+ return hpx.command.DownloadInfo(
+ identifier = EX_IDENTIFIER,
+ name = "ExHentai",
+ parser = website_url_regex_gen("exhentai.org", path_regex=r"g\/[0-9]{3,10}\/[0-9a-zA-Z]{3,15}", trailing_slash=True, variable_tld=False, trailing_fragment=True, end=True),
+ sites = ("https://exhentai.org",),
+ description = "Download manga and doujinshi from exhentai.org",
+ )
+
+@hpx.attach("Download.query", trigger=EH_IDENTIFIER)
+def eh_download_query(item):
+ return download_query(item, False)
+
+@hpx.attach("Download.query", trigger=EX_IDENTIFIER)
+def ex_download_query(item):
+ return download_query(item, True)
+
+def download_query(item, is_exhentai):
+ """
+ Called to query for resource URLs that should be downloaded.
+ Note that HPX will handle the actual downloading part.
+ The attached handler should just return all the URLs that should be downloaded in the form of .:class:`DownloadRequest` objects
+
+ should return:
+ a tuple of :class:`DownloadRequest` for all the URL resources that should be downloaded.
+ Note that the download system is recursive, so if the URL resource matches a download handler (the same or a different one),
+ That handler will be called upon with a new :class:`DownloadItem` for that particular URL
+ (though only once, meaning, no handler will be called upon again with the exact same URL during a single session)
+ """
+
+ gid, gtoken = parse_url(item.url)
+ download_requests = []
+
+ #===============================================================================
+ # get ehentai login
+ login_site = URLS['ex'] if is_exhentai else URLS['eh']
+ login_status = hpx.command.GetLoginStatus(login_site)
+ login_session = None
+ if login_status:
+ login_session = hpx.command.GetLoginSession(login_site)
+ if not login_session:
+ log.warning("unable to get a login sesion for querying gallery data")
+ return ()
+ log.info("logged in, attempting to download archive")
+
+ #===============================================================================
+ # get the gallery metadata which should have the archive key
+ # https://ehwiki.org/wiki/API#Gallery_Metadata
+ log.info("getting archiver key")
+ eh_data = {
+ 'method': 'gdata',
+ 'gidlist': [[gid, gtoken]],
+ }
+ req_props = hpx.command.RequestProperties(
+ headers=HEADERS,
+ json=eh_data,
+ session=login_session
+ )
+ api_url = URLS['ex_api' if is_exhentai else 'e_api']
+ log.info(f"requesting with api url {api_url}")
+ r = hpx.command.SinglePOSTRequest().request(api_url, req_props)
+ if not r.ok:
+ log.warning(f"got invalid metadata page or bad status: {r.status_code}")
+ log.debug(r.text)
+ return ()
+ try:
+ response = r.json
+ except json.JSONDecodeError:
+ response = None
+ log.info("got empty response when trying to retrieve archiver key, this usually means that user has no access to exhentai")
+ return ()
+ if not response or 'error' in response:
+ log.warning("response has an error of some sort, and so we have no archive key to use")
+ log.debug(r.text)
+ return ()
+
+ #===============================================================================
+ # Read the metadata of the gallery to fill out the download queue item
+ #
+ # While in theory we should only ever have a single entry in the response, best keep it more general just in case this gets retrofitted to handle multiple urls at once
+ # The information we reliably get is the thumbnail url and the title of the gallery
+ # We seem to also always get an archive key, but the key is not always valid, and so the archive url request can fail
+ #
+ # Yes, there is an expunged flag in the metadata, but it is always false when the gallery/archive is not available
+ # It is also true sometimes and yet the gallery/archive is totally accessable and so is meaningless
+ for gdata in response['gmetadata']:
+ archive_req = False
+ try:
+ if 'title' in gdata:
+ item.name = gdata['title']
+ if 'thumb' in gdata:
+ download_requests.append(
+ DownloadRequest(
+ downloaditem=item,
+ url=gdata['thumb'],
+ is_thumbnail=True,
+ properties=hpx.command.RequestProperties(method=hpx.Method.GET, headers=HEADERS, session=login_session), # we need to use the same session
+ ))
+ if 'archiver_key' in gdata:
+ log.info(f"found archiver key for gallery {(gid, gtoken)}")
+ a_key = gdata['archiver_key']
+ a_url = URLS['ex_archiver' if is_exhentai else 'e_archiver'].format(gallery_id=gid, gallery_token=gtoken, archiver_key=a_key)
+ form_data = {
+ "dltype": "org", #original quality, instead of a resampled version
+ "dlcheck": "Download Original Archive"
+ }
+ req_props = hpx.command.RequestProperties(
+ headers=HEADERS,
+ data=form_data,
+ session=login_session
+ )
+ r = hpx.command.SinglePOSTRequest().request(a_url, req_props)
+ if r.ok:
+ if "Insufficient funds" in r.text:
+ log.info("Unable to grab gallery archive due to insufficent funds (GP) on the account")
+ item.name = "(Insufficient GP) "+item.name
+ elif "Key missing, or incorrect key provided" not in r.text:
+ soup = BeautifulSoup(r.text, "html.parser")
+ dp_url = soup.find("p", id="continue")
+ if dp_url and dp_url.a: # finally
+ download_requests.append(
+ DownloadRequest(
+ downloaditem=item,
+ url=dp_url.a['href'] + '?start=1',
+ properties=hpx.command.RequestProperties(method=hpx.Method.GET, headers=HEADERS, session=login_session), # we need to use the same session
+ filename=item.name.strip()+'.zip'))
+ archive_req = True
+ log.debug(f"adding the archive url {download_requests[-1].url}")
+ if not archive_req:
+ log.info("Something went wrong and we did not actually find a URL")
+ #TODO Actually better handle the various cases of why we do not have a url
+ else:
+ log.warning(f"got invalid key page or bad status: {r.status_code}")
+ if r.status_code == 404 and "This gallery is currently unavailable" in r.text:
+ #We know that there is a valid key for us to get here, so the gallery existed at some point in the past
+ #This seems like it is most of the time a copyright takedown, but I have no idea why this is not marked as expunged
+ item.name = "(Gallery Unavailable) "+item.name
+ else:
+ log.warning(f"didn't find archiver key for data: {eh_data}")
+ item.name = "(Archive Unavailable) "+item.name
+ except Exception as e:
+ log.debug(f"got an error, last request content: \n\t {r.text}")
+ raise
+
+ if not archive_req:
+ pass
+ # TODO: download individual images instead
+
+ if download_requests:
+ log.info(f"was able to prepare {len(download_requests)} requests")
+ else:
+ log.info("unable to prepare any URLs to download")
+ return tuple(download_requests)
+
+@hpx.attach("Download.done", trigger=[EX_IDENTIFIER, EH_IDENTIFIER])
+def download_done(result):
+ """
+ Called when downloading of all :class:`DownloadRequest` for a specific :class:`DownloadItem` has finished.
+ The handler should do any post-processing here (archive files, rename files or folders, delete extranous files and etc.).
+ Remember to set the `status` property on the :class:`DownloadResult` object to `False` if the post-processing was a failure.
+ Note that the handler should *not* import the file into HPX (if it's an item), that part will be taken care of by HPX
+
+ should return:
+ the same :class:`DownloadResult` that was provided to the handler, potentially modified on the 'path' or `status` and `reason` properties
+ """
+ # there's nothing special to post-process in the case of e(x)hentai downloader, so just return the result as is
+ log.info(f"download of archive was successful for {result.downloaditem.name}")
+ #TODO Mark it as a failure if there was only a thumbnail to download
+ #TODO Archive the individual images together into a cbz or something if we grabbed individual images
+ return result
+
+def parse_url(url):
+ "Parses url into a tuple of gallery id and token"
+ gallery_id = None
+ gallery_token = None
+
+ gallery_id_token = regex.search('(?<=g/)([0-9]+)/([a-zA-Z0-9]+)', url)
+ if gallery_id_token:
+ gallery_id_token = gallery_id_token.group()
+ gallery_id, gallery_token = gallery_id_token.split('/')
+ else:
+ log.warning("Error extracting g_id and g_token from url: {}".format(url))
+ return int(gallery_id), gallery_token
diff --git a/plugins/EHentai Downloader/readme.md b/plugins/EHentai Downloader/readme.md
new file mode 100644
index 0000000..36198e7
--- /dev/null
+++ b/plugins/EHentai Downloader/readme.md
@@ -0,0 +1,22 @@
+EHentai Downloader
+----------------------------
+
+> A plugin that enables downloading manga and doujinshi from E-Hentai & ExHentai
+
+**IMPORTANT:** To download using GP/credits, the plugin [EHentai Login](https://github.com/happypandax/plugins/tree/master/plugins/EHentai%20Login) is required to be present
+
+**IMPORTANT:** Only downloading through ehentai's archiver system (which costs GP/credits) is supported at this time
+
+## Configuration
+
+There's no available config options for this plugin
+
+## Things yet to be implemented
+
+- Support scraping the individual images
+- Torrents (waiting for HPX to support this)
+
+# Changelog
+
+- `1.0.0`
+ - first version
\ No newline at end of file
diff --git a/plugins/EHentai Downloader/test.py b/plugins/EHentai Downloader/test.py
new file mode 100644
index 0000000..36a317e
--- /dev/null
+++ b/plugins/EHentai Downloader/test.py
@@ -0,0 +1 @@
+# test.py
\ No newline at end of file
diff --git a/plugins/EHentai Login/hplugin.json b/plugins/EHentai Login/hplugin.json
new file mode 100644
index 0000000..f35d421
--- /dev/null
+++ b/plugins/EHentai Login/hplugin.json
@@ -0,0 +1,16 @@
+{
+ "id": "d9b1d111-7250-4083-9efb-356fabbeada7",
+ "shortname": "ehentai-login",
+ "name": "EHentai Login",
+ "version": "1.1.0",
+ "description": "A plugin that can login to E-Hentai & ExHentai",
+ "author": "Twiddly",
+ "site_folder": "site/",
+ "update_url": "https://github.com/happypandax/plugins/tree/master/plugins/EHentai%20Login",
+ "website": "https://github.com/happypandax/plugins/tree/master/plugins/EHentai%20Login",
+ "entry": "main.py",
+ "test": "test.py",
+ "require": [
+ "happypandax >= 0.10.0"
+ ]
+}
diff --git a/plugins/EHentai Login/main.py b/plugins/EHentai Login/main.py
new file mode 100644
index 0000000..7563dbe
--- /dev/null
+++ b/plugins/EHentai Login/main.py
@@ -0,0 +1,191 @@
+# main.py
+import __hpx__ as hpx
+import pickle
+import os
+
+from bs4 import BeautifulSoup
+
+log = hpx.get_logger("main")
+
+
+
+current_user_name = ""
+status_text = ""
+response = None
+user_dict = None
+
+save_file = os.path.join(hpx.constants.current_dir, '.info')
+
+default_delay = 8
+
+HEADERS = {'user-agent':"Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0"}
+
+match_url_prefix = r"^(http\:\/\/|https\:\/\/)?(www\.)?" # http:// or https:// + www.
+match_url_end = r"\/?$"
+
+url_regex = match_url_prefix + r"((exhentai|(g\.)?e-hentai)\.org)" + match_url_end
+
+MAIN_URLS = {
+ 'eh': "https://e-hentai.org",
+ 'ex': "https://exhentai.org"
+}
+
+URLS = MAIN_URLS
+URLS.update({
+ 'login': "https://e-hentai.org/home.php"
+})
+
+@hpx.subscribe("init")
+def inited():
+ # set default delay values if not set
+ delays = hpx.get_setting("network", "delays", {})
+ for u in (MAIN_URLS['ex'], MAIN_URLS['eh']):
+ if u not in delays:
+ log.info(f"Setting delay on {u} requests to {default_delay}")
+ delays[u] = default_delay
+ hpx.update_setting("network", "delays", delays)
+
+ # retrieve saved user info
+ if os.path.exists(save_file):
+ with open(save_file, "rb") as f:
+ user_dict = pickle.load(f)
+ if user_dict:
+ login(user_dict, {})
+ if response is not None:
+ log.info("Successfully re-logged in")
+
+@hpx.subscribe("disable")
+def disabled():
+ save_user_dict()
+
+@hpx.subscribe("remove")
+def removed():
+ pass
+
+@hpx.attach("Login.info")
+def login_info():
+ return hpx.command.LoginInfo(
+ identifier = "ehentai",
+ name = "EHentai",
+ parser = url_regex,
+ sites = ("www.e-hentai.org", "www.exhentai.org"),
+ description = "Login to E-Hentai & ExHentai",
+ )
+
+@hpx.attach("Login.login", trigger="ehentai")
+def login(userpass, options):
+ global current_user_name
+ global status_text
+ global response
+ global user_dict
+
+ user_dict = userpass
+ response = None
+ current_user_name = ""
+
+ ipb_member = userpass.get('ipb_member_id', "")
+ ipb_pass = userpass.get('ipb_pass_hash', "")
+ try:
+ if not ipb_member or not ipb_pass:
+ raise ValueError("Missing ipb_member_id or ipb_pass_hash")
+
+ cookies = {}
+
+ additional = userpass.get('additional', "")
+ if additional:
+ try:
+ additional = {k.strip():v.strip() for k, v in [x.strip().split('=', 1) for x in additional.split(',')] }
+ cookies.update(additional)
+ except:
+ raise ValueError("Failed to parse additional values")
+
+ cookies.update({
+ 'ipb_member_id': ipb_member,
+ 'ipb_pass_hash': ipb_pass,
+ })
+
+ # prepare request
+ req_props = hpx.command.RequestProperties(
+ session=True,
+ cookies=cookies,
+ headers=HEADERS
+ )
+
+ req = hpx.command.SingleGETRequest()
+
+ # check ehentai.org/home.php
+ r = req.request(URLS['login'], req_props)
+
+ if r.ok:
+ bad_access, msg = check_access(r)
+ status_text = msg
+ if not bad_access:
+ if userpass.get("exhentai", True):
+ # check exhentai
+ req_props.session = r.session
+ r = req.request(URLS['ex'], req_props)
+ if r.ok:
+ bad_access, status_text = check_access(r, ex=True)
+ else:
+ status_text = "Could not access ExHentai"
+
+ response = r
+
+ current_user_name = ipb_member
+ save_user_dict()
+
+ else:
+ status_text = r.reason
+
+ except ValueError as e:
+ status_text = str(e)
+
+ return response
+
+@hpx.attach("Login.status", trigger="ehentai")
+def status(options):
+ return status_text
+
+@hpx.attach("Login.logged_in", trigger="ehentai")
+def logged_in(options):
+ if response:
+ return True
+ return False
+
+@hpx.attach("Login.response", trigger="ehentai")
+def response_(options):
+ return response
+
+@hpx.attach("Login.current_user", trigger="ehentai")
+def current_user(options):
+ return current_user_name
+
+def check_access(r, ex=False):
+ msg = ""
+ bad_access = False
+ content_type = r.headers['content-type']
+ text = r.text
+ if 'image/gif' in content_type:
+ msg = "No access to ExHentai"
+ elif 'text/html' and 'Your IP address has been' in text:
+ msg = text
+ bad_access = True
+
+ if not bad_access and not ex:
+ soup = BeautifulSoup(text, "html.parser")
+ if soup.find("div", class_="homebox"): # we have access to home.php
+ pass
+ elif soup.find("form"): # login page
+ bad_access = True
+ msg = "Wrong credentials!"
+ if msg:
+ log.info(f"MSG: {msg}")
+ return bad_access, msg
+
+def save_user_dict():
+ global user_dict
+
+ # save user info
+ if user_dict:
+ with open(save_file, "wb") as f:
+ user_dict = pickle.dump(user_dict, f)
diff --git a/plugins/EHentai Login/readme.md b/plugins/EHentai Login/readme.md
new file mode 100644
index 0000000..7258ebc
--- /dev/null
+++ b/plugins/EHentai Login/readme.md
@@ -0,0 +1,22 @@
+EHentai Login
+----------------------------
+
+> This plugin can log-in to E-Hentai & ExHentai
+
+To login, go to this plugin's site through HPX.
+The plugin site can be accessed through a webclient, or at `[webserver-host]/plugin/d9b1d111-7250-4083-9efb-356fabbeada7`.
+Replace `[webserver-host]` with wherever your HPX webserver is hosted at.
+
+# Changelog
+
+- `1.1.0`
+ - Added a new field for additional cookie values
+
+- `1.0.0`
+ - Updated to reflect new changes in HPX v0.10.0
+
+- `0.2.0b`
+ - increase default delay limit on EH requests to `9` from `4` secs, this value can be tweaked in `network.delays` inside your`config.yaml`
+
+- `0.1.0b`
+ - first version
diff --git a/plugins/EHentai Login/site/index.html b/plugins/EHentai Login/site/index.html
new file mode 100644
index 0000000..6d463c0
--- /dev/null
+++ b/plugins/EHentai Login/site/index.html
@@ -0,0 +1,62 @@
+{% extends __default__ %}
+
+{% block include_javascript %}
+
+{% endblock %}
+
+{% block title %}
+EHentai Metadata Plugin
+{% endblock %}
+
+{% block before_content_form %}
+Toggle the ExHentai option before logging in to also check for ExHentai access.
+The user has access to both E-Hentai and ExHentai if no message is displayed on a succesful log-in. Else the message No access to ExHentai will be displayed.
+
+
To find your IPB Member ID and IPB Pass Hash, follow these steps (should work on all browsers):
+
+ - Navigate to e-hentai.org (needs to be logged in) or exhentai.org
+ - Right click on page => Inpect element
+ - Go on Console tab
+ - Write: document.cookie
+ - A line of values should appear that correspond to active cookies
+ - Look for the ipb_member_id, ipb_pass_hash and other values
+
+