mirrors/aniyomi-extensions
1
0
Fork 0
mirror of https://github.com/Dark25/aniyomi-extensions.git synced 2024-10-28 08:09:37 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(lib/UniversalExtractor): Implementation of a universal extractor

Browse source
This commit is contained in:
Dark25 2024-09-11 23:15:56 +02:00
parent ed8666793a
commit 23194eab9a
7 changed files with 114 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
lib/playlist-utils/src/main/java/eu/kanade/tachiyomi/lib/playlistutils/PlaylistUtils.kt
View file

@ -8,6 +8,7 @@ import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import okhttp3.internal.commonEmptyHeaders
import kotlin.math.abs
class PlaylistUtils(private val client: OkHttpClient, private val headers: Headers = commonEmptyHeaders) {
@ -135,7 +136,7 @@ class PlaylistUtils(private val client: OkHttpClient, private val headers: Heade
val resolution = it.substringAfter("RESOLUTION=")
.substringBefore("\n")
.substringAfter("x")
.substringBefore(",") + "p"
.substringBefore(",").let(::stnQuality)
val videoUrl = it.substringAfter("\n").substringBefore("\n").let { url ->
getAbsoluteUrl(url, playlistUrl, masterUrlBasePath)?.trimEnd()
@ -334,6 +335,13 @@ class PlaylistUtils(private val client: OkHttpClient, private val headers: Heade
// ============================= Utilities ==============================
private fun stnQuality(quality: String): String {
val intQuality = quality.toInt()
val standardQualities = listOf(144, 240, 360, 480, 720, 1080)
val result = standardQualities.minByOrNull { abs(it - intQuality) } ?: quality
return "${result}p"
}
companion object {
private const val PLAYLIST_SEPARATOR = "#EXT-X-STREAM-INF:"

7
lib/universal-extractor/build.gradle.kts Normal file
View file

@ -0,0 +1,7 @@
plugins {
id("lib-android")
}
dependencies {
implementation(project(":lib:playlist-utils"))
}

87
lib/universal-extractor/src/main/java/eu/kanade/tachiyomi/lib/universalextractor/UniversalExtractor.kt Normal file
View file

@ -0,0 +1,87 @@
package eu.kanade.tachiyomi.lib.universalextractor
import android.annotation.SuppressLint
import android.app.Application
import android.os.Handler
import android.os.Looper
import android.util.Log
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebView
import android.webkit.WebViewClient
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import uy.kohesive.injekt.injectLazy
import java.util.Locale
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
class UniversalExtractor(private val client: OkHttpClient) {
private val context: Application by injectLazy()
private val handler by lazy { Handler(Looper.getMainLooper()) }
@SuppressLint("SetJavaScriptEnabled")
fun videosFromUrl(origRequestUrl: String, origRequestHeader: Headers, customQuality: String? = null, prefix: String = ""): List<Video> {
val host = origRequestUrl.toHttpUrl().host.substringBefore(".").proper()
val latch = CountDownLatch(1)
var webView: WebView? = null
var resultUrl = ""
val playlistUtils by lazy { PlaylistUtils(client, origRequestHeader) }
val headers = origRequestHeader.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap()
handler.post {
val newView = WebView(context)
webView = newView
with(newView.settings) {
javaScriptEnabled = true
domStorageEnabled = true
databaseEnabled = true
useWideViewPort = false
loadWithOverviewMode = false
userAgentString = origRequestHeader["User-Agent"]
}
newView.webViewClient = object : WebViewClient() {
override fun shouldInterceptRequest(
view: WebView,
request: WebResourceRequest,
): WebResourceResponse? {
val url = request.url.toString()
if (VIDEO_REGEX.containsMatchIn(url)) {
resultUrl = url
latch.countDown()
}
return super.shouldInterceptRequest(view, request)
}
}
webView?.loadUrl(origRequestUrl, headers)
}
latch.await(TIMEOUT_SEC, TimeUnit.SECONDS)
handler.post {
webView?.stopLoading()
webView?.destroy()
webView = null
}
Log.d("UniversalExtractor", "Result URL: $resultUrl from $origRequestUrl host: $host")
return when {
"m3u8" in resultUrl -> playlistUtils.extractFromHls(resultUrl, origRequestUrl, videoNameGen = { "$prefix - $host: $it" })
"mpd" in resultUrl -> playlistUtils.extractFromDash(resultUrl, { it -> "$prefix - $host: $it" }, referer = origRequestUrl)
"mp4" in resultUrl -> Video(resultUrl, "$prefix - $host: ${customQuality ?: "Mirror"}", resultUrl, origRequestHeader.newBuilder().add("referer", origRequestUrl).build()).let(::listOf)
else -> emptyList()
}
}
private fun String.proper(): String {
return this.replaceFirstChar { if (it.isLowerCase()) it.titlecase(
Locale.getDefault()) else it.toString() }
}
companion object {
const val TIMEOUT_SEC: Long = 20
private val VIDEO_REGEX by lazy { Regex("\\.(mp4|m3u8|mpd)") }
}
}

3
src/es/animeflv/build.gradle
View file

@ -1,7 +1,7 @@
ext {
extName = 'AnimeFLV'
extClass = '.AnimeFlv'
extVersionCode = 56
extVersionCode = 57
}
apply from: "$rootDir/common.gradle"
@ -11,4 +11,5 @@ dependencies {
implementation(project(':lib:streamtape-extractor'))
implementation(project(':lib:okru-extractor'))
implementation(project(':lib:streamwish-extractor'))
implementation(project(':lib:universal-extractor'))
}

8
src/es/animeflv/src/eu/kanade/tachiyomi/animeextension/es/animeflv/AnimeFlv.kt
View file

@ -14,6 +14,7 @@ import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
import eu.kanade.tachiyomi.lib.streamtapeextractor.StreamTapeExtractor
import eu.kanade.tachiyomi.lib.streamwishextractor.StreamWishExtractor
import eu.kanade.tachiyomi.lib.universalextractor.UniversalExtractor
import eu.kanade.tachiyomi.lib.youruploadextractor.YourUploadExtractor
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
@ -109,18 +110,19 @@ class AnimeFlv : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
private val okruExtractor by lazy { OkruExtractor(client) }
private val yourUploadExtractor by lazy { YourUploadExtractor(client) }
private val streamWishExtractor by lazy { StreamWishExtractor(client, headers.newBuilder().add("Referer", "$baseUrl/").build()) }
private val universalExtractor by lazy { UniversalExtractor(client) }
override fun videoListParse(response: Response): List<Video> {
val document = response.asJsoup()
val jsonString = document.selectFirst("script:containsData(var videos = {)")?.data() ?: return emptyList()
val responseString = jsonString.substringAfter("var videos =").substringBefore(";").trim()
return json.decodeFromString<ServerModel>(responseString).sub.parallelCatchingFlatMapBlocking {
return json.decodeFromString<ServerModel>(responseString).sub.parallelCatchingFlatMapBlocking { it ->
when (it.title) {
"Stape" -> listOf(streamTapeExtractor.videoFromUrl(it.url ?: it.code)!!)
"Okru" -> okruExtractor.videosFromUrl(it.url ?: it.code)
"YourUpload" -> yourUploadExtractor.videoFromUrl(it.url ?: it.code, headers = headers)
"SW" -> streamWishExtractor.videosFromUrl(it.url ?: it.code, videoNameGen = { "StreamWish:$it" })
else -> emptyList()
"SW" -> streamWishExtractor.videosFromUrl(it.url ?: it.code, videoNameGen = {"StreamWish:$it"})
else -> universalExtractor.videosFromUrl(it.url ?: it.code, headers)
}
}
}

3
src/es/gnula/build.gradle
View file

@ -1,7 +1,7 @@
ext {
extName = 'Gnula'
extClass = '.Gnula'
extVersionCode = 24
extVersionCode = 25
}
apply from: "$rootDir/common.gradle"
@ -21,4 +21,5 @@ dependencies {
implementation(project(':lib:streamhidevid-extractor'))
implementation(project(':lib:filemoon-extractor'))
implementation(project(':lib:streamwish-extractor'))
implementation(project(':lib:universal-extractor'))
}

47
src/es/gnula/src/eu/kanade/tachiyomi/animeextension/es/gnula/Gnula.kt
View file

@ -12,25 +12,11 @@ import eu.kanade.tachiyomi.animesource.model.SAnime
import eu.kanade.tachiyomi.animesource.model.SEpisode
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
import eu.kanade.tachiyomi.lib.burstcloudextractor.BurstCloudExtractor
import eu.kanade.tachiyomi.lib.doodextractor.DoodExtractor
import eu.kanade.tachiyomi.lib.fastreamextractor.FastreamExtractor
import eu.kanade.tachiyomi.lib.filemoonextractor.FilemoonExtractor
import eu.kanade.tachiyomi.lib.mp4uploadextractor.Mp4uploadExtractor
import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
import eu.kanade.tachiyomi.lib.streamhidevidextractor.StreamHideVidExtractor
import eu.kanade.tachiyomi.lib.streamlareextractor.StreamlareExtractor
import eu.kanade.tachiyomi.lib.streamtapeextractor.StreamTapeExtractor
import eu.kanade.tachiyomi.lib.streamwishextractor.StreamWishExtractor
import eu.kanade.tachiyomi.lib.upstreamextractor.UpstreamExtractor
import eu.kanade.tachiyomi.lib.uqloadextractor.UqloadExtractor
import eu.kanade.tachiyomi.lib.voeextractor.VoeExtractor
import eu.kanade.tachiyomi.lib.youruploadextractor.YourUploadExtractor
import eu.kanade.tachiyomi.lib.universalextractor.UniversalExtractor
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
import eu.kanade.tachiyomi.util.parallelCatchingFlatMapBlocking
import kotlinx.serialization.json.Json
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Request
import okhttp3.Response
import org.jsoup.nodes.Document
@ -162,36 +148,7 @@ class Gnula : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
private fun serverVideoResolver(url: String, prefix: String = ""): List<Video> {
val embedUrl = url.lowercase()
return when {
embedUrl.contains("voe") -> VoeExtractor(client).videosFromUrl(url, prefix)
embedUrl.contains("ok.ru") || embedUrl.contains("okru") -> OkruExtractor(client).videosFromUrl(url, prefix)
embedUrl.contains("filemoon") || embedUrl.contains("moonplayer") -> {
val vidHeaders = headers.newBuilder()
.add("Origin", "https://${url.toHttpUrl().host}")
.add("Referer", "https://${url.toHttpUrl().host}/")
.build()
FilemoonExtractor(client).videosFromUrl(url, prefix = "$prefix Filemoon:", headers = vidHeaders)
}
embedUrl.contains("uqload") -> UqloadExtractor(client).videosFromUrl(url, prefix = prefix)
embedUrl.contains("mp4upload") -> Mp4uploadExtractor(client).videosFromUrl(url, headers, prefix = prefix)
embedUrl.contains("wishembed") || embedUrl.contains("streamwish") || embedUrl.contains("strwish") || embedUrl.contains("wish") -> {
val docHeaders = headers.newBuilder()
.add("Origin", "https://streamwish.to")
.add("Referer", "https://streamwish.to/")
.build()
StreamWishExtractor(client, docHeaders).videosFromUrl(url, videoNameGen = { "$prefix StreamWish:$it" })
}
embedUrl.contains("doodstream") || embedUrl.contains("dood.") || embedUrl.contains("ds2play") || embedUrl.contains("doods.") -> {
val url2 = url.replace("https://doodstream.com/e/", "https://dood.to/e/")
listOf(DoodExtractor(client).videosFromUrl(url2, quality = prefix, redirect = false)).flatten()
}
embedUrl.contains("streamlare") -> StreamlareExtractor(client).videosFromUrl(url, prefix = prefix)
embedUrl.contains("yourupload") || embedUrl.contains("upload") -> YourUploadExtractor(client).videoFromUrl(url, headers = headers, prefix = prefix)
embedUrl.contains("burstcloud") || embedUrl.contains("burst") -> BurstCloudExtractor(client).videoFromUrl(url, headers = headers, prefix = prefix)
embedUrl.contains("fastream") -> FastreamExtractor(client, headers).videosFromUrl(url, prefix = "$prefix Fastream:")
embedUrl.contains("upstream") -> UpstreamExtractor(client).videosFromUrl(url, prefix = prefix)
embedUrl.contains("streamtape") || embedUrl.contains("stp") || embedUrl.contains("stape") -> listOf(StreamTapeExtractor(client).videoFromUrl(url, quality = "$prefix StreamTape")!!)
embedUrl.contains("ahvsh") || embedUrl.contains("streamhide") || embedUrl.contains("guccihide") || embedUrl.contains("streamvid") -> StreamHideVidExtractor(client).videosFromUrl(url, "$prefix StreamHide")
else -> emptyList()
else -> UniversalExtractor(client).videosFromUrl(url, headers, prefix = prefix)
}
}