Skip to content
This repository has been archived by the owner on Jan 15, 2024. It is now read-only.

Commit

Permalink
Fixed Google Play parsing. (#165)
Browse files Browse the repository at this point in the history 
* Fixed Google Play parsing.
Also updated some build parts to fix Android Studio complaining.

Fixes #164

* Travis emulator

* Travis emulator
  • Loading branch information
@d4rken
d4rken authored Jul 28, 2018
1 parent a4f7b38 commit 0376cd4
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 94 deletions.
3 changes: 2 additions & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@ android:
components:
- platform-tools
- tools
- build-tools-25.0.3
- build-tools-27.0.3
- android-22
- android-25
- android-27
- extra-google-m2repository
- extra-android-m2repository
- sys-img-armeabi-v7a-android-22
Expand Down
124 changes: 62 additions & 62 deletions app/build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
apply plugin: 'com.android.application'
apply plugin: 'realm-android'
apply plugin: 'me.tatarka.retrolambda'
apply plugin: 'com.github.kt3k.coveralls'
apply plugin: 'jacoco'

Expand All @@ -13,10 +12,12 @@ def gitSha() {
return p.text.trim()
}

def packageName = "subreddit.android.appstore"

android {
def signingPropFile = new File(System.properties['user.home'], ".signing/reddit-android-appstore/signing.properties")
def signingPropFile = new File(System.properties['user.home'], ".signing/${packageName}/signing.properties")
if (signingPropFile.canRead()) {
def Properties signingProps = new Properties()
Properties signingProps = new Properties()
signingProps.load(new FileInputStream(signingPropFile))
signingConfigs {
release {
Expand All @@ -28,19 +29,19 @@ android {
}
}

compileSdkVersion 25
buildToolsVersion "25.0.3"
compileSdkVersion 27
buildToolsVersion '27.0.3'

def versionMajor = 0
def versionMinor = 8
def versionPatch = 1
def versionMinor = 9
def versionPatch = 0
def versionBuild = 0


defaultConfig {
applicationId "subreddit.android.appstore"
applicationId "${packageName}"
minSdkVersion 16
targetSdkVersion 25
targetSdkVersion 27
versionCode versionMajor * 10000 + versionMinor * 1000 + versionPatch * 100 + versionBuild
versionName "${versionMajor}.${versionMinor}.${versionPatch}"
}
Expand All @@ -65,13 +66,12 @@ android {
applicationVariants.all { variant ->
if (variant.buildType.name != "debug") {
variant.outputs.each { output ->
def file = output.outputFile
//noinspection GroovyAssignabilityCheck
output.outputFile = new File(file.parent, "rAndroid_AppStore-v" + defaultConfig.versionName + "(" + defaultConfig.versionCode + ")-" + variant.buildType.name.toUpperCase() + "-" + gitSha() + ".apk")
output.outputFileName = "${packageName}-v" + defaultConfig.versionName + "(" + defaultConfig.versionCode + ")-" + variant.buildType.name.toUpperCase() + "-" + gitSha() + ".apk"
}
}
}
}
flavorDimensions "default"
productFlavors {
mock {
applicationIdSuffix = ".mock"
Expand All @@ -85,7 +85,7 @@ android {
// Remove mockRelease as it's not needed.
android.variantFilter { variant ->
if (variant.buildType.name.equals('release') && variant.getFlavors().get(0).name.equals('mock')) {
variant.setIgnore(true);
variant.setIgnore(true)
}
}

Expand All @@ -108,7 +108,7 @@ android {
}
}

task jacocoTestReport(type: JacocoReport, dependsOn: ['testProdDebugUnitTest' , 'createProdDebugCoverageReport']) {
task jacocoTestReport(type: JacocoReport, dependsOn: ['testProdDebugUnitTest', 'createProdDebugCoverageReport']) {

reports {
xml.enabled = true
Expand Down Expand Up @@ -155,7 +155,7 @@ tasks.withType(Test) {
systemProperty "robolectric.logging", "stdout"
}
ext {
supportLibVersion = '25.3.1'
supportLibVersion = '27.1.1'
}
//noinspection GroovyAssignabilityCheck
configurations.all {
Expand All @@ -165,83 +165,83 @@ configurations.all {
resolutionStrategy.force "com.android.support:support-v4:${supportLibVersion}"
resolutionStrategy.force "com.android.support:appcompat-v7:${supportLibVersion}"
resolutionStrategy.force "com.android.support:design:${supportLibVersion}"
resolutionStrategy.force 'com.google.code.findbugs:jsr305:1.3.9'
}
dependencies {
compile fileTree(include: ['*.jar'], dir: 'libs')

// jUnit
testCompile 'junit:junit:4.12'

// Support libs
compile "com.android.support:support-annotations:${supportLibVersion}"
compile "com.android.support:appcompat-v7:${supportLibVersion}"
compile "com.android.support:recyclerview-v7:${supportLibVersion}"
compile "com.android.support:customtabs:${supportLibVersion}"
compile "com.android.support:design:${supportLibVersion}"
compile "com.android.support:cardview-v7:${supportLibVersion}"
compile "com.android.support:preference-v14:${supportLibVersion}"
compile "com.android.support:design:${supportLibVersion}"

// JavaPoet
compile 'com.squareup:javapoet:1.9.0'
implementation "com.android.support:support-annotations:${supportLibVersion}"
implementation "com.android.support:appcompat-v7:${supportLibVersion}"
implementation "com.android.support:recyclerview-v7:${supportLibVersion}"
implementation "com.android.support:customtabs:${supportLibVersion}"
implementation "com.android.support:design:${supportLibVersion}"
implementation "com.android.support:cardview-v7:${supportLibVersion}"
implementation "com.android.support:preference-v14:${supportLibVersion}"
implementation "com.android.support:design:${supportLibVersion}"

// Recyclerview fast scroller
compile 'com.futuremind.recyclerfastscroll:fastscroll:0.2.4'
implementation 'com.futuremind.recyclerfastscroll:fastscroll:0.2.4'

// OKHttp
compile 'com.squareup.okhttp3:okhttp:3.4.1'
compile 'com.squareup.okhttp3:logging-interceptor:3.5.0'
implementation 'com.squareup.okhttp3:okhttp:3.10.0'
implementation 'com.squareup.okhttp3:logging-interceptor:3.10.0'

// Retrofit
compile 'com.squareup.retrofit2:retrofit:2.1.0'
compile 'com.squareup.retrofit2:converter-gson:2.1.0'
compile 'com.jakewharton.retrofit:retrofit2-rxjava2-adapter:1.0.0-RC2'
implementation 'com.squareup.retrofit2:retrofit:2.4.0'
implementation 'com.squareup.retrofit2:converter-gson:2.2.0'
implementation 'com.jakewharton.retrofit:retrofit2-rxjava2-adapter:1.0.0-RC2'

// Glide
compile 'com.github.bumptech.glide:glide:4.0.0'
annotationProcessor 'com.github.bumptech.glide:compiler:4.0.0'
implementation 'com.github.bumptech.glide:glide:4.3.1'
annotationProcessor 'com.github.bumptech.glide:compiler:4.3.1'

// Flow layout for tags
compile 'com.wefika:flowlayout:0.4.0'
implementation 'com.wefika:flowlayout:0.4.0'

// RX
compile 'io.reactivex.rxjava2:rxandroid:2.0.0-RC1'
compile 'io.reactivex.rxjava2:rxjava:2.0.0-RC2'
implementation 'io.reactivex.rxjava2:rxandroid:2.0.2'
implementation 'io.reactivex.rxjava2:rxjava:2.1.16'

// Gson
compile 'com.google.code.gson:gson:2.7'
implementation 'com.google.code.gson:gson:2.8.0'

//Dagger
compile 'com.google.dagger:dagger:2.11'
annotationProcessor "com.google.dagger:dagger-compiler:2.11"
provided 'javax.annotation:jsr250-api:1.0'
implementation 'com.google.dagger:dagger:2.13'
annotationProcessor "com.google.dagger:dagger-compiler:2.13"

//Butterknife
compile 'com.jakewharton:butterknife:8.0.1'
annotationProcessor 'com.jakewharton:butterknife-compiler:8.0.1'
implementation 'com.jakewharton:butterknife:8.8.1'
annotationProcessor 'com.jakewharton:butterknife-compiler:8.8.1'

//Timber
compile 'com.jakewharton.timber:timber:4.1.2'
// Timber
implementation 'com.jakewharton.timber:timber:4.7.0'

// Hamcrest
testCompile 'org.hamcrest:hamcrest-library:1.3'
testImplementation 'org.hamcrest:hamcrest-library:1.3'

// Mockito
testCompile 'org.mockito:mockito-core:2.8.9'
testImplementation 'org.mockito:mockito-core:2.20.0'

// jUnit
testImplementation 'junit:junit:4.12'

// jsoup HTML parser library @ https://jsoup.org/
implementation 'org.jsoup:jsoup:1.11.3'

//Intrumentation tests (UI tests)
androidTestCompile 'com.android.support.test.espresso:espresso-core:2.2.2'
androidTestCompile 'com.android.support.test.espresso:espresso-contrib:2.2.2'
// Intrumentation tests (UI tests)
androidTestImplementation "com.android.support.test:runner:1.0.2"
androidTestImplementation "com.android.support.test:rules:1.0.2"
androidTestImplementation "com.android.support.test.espresso:espresso-core:3.0.2"
androidTestImplementation 'com.android.support.test.espresso:espresso-contrib:3.0.2'
androidTestImplementation 'com.android.support.test.espresso:espresso-intents:3.0.2'
androidTestImplementation 'com.android.support.test.espresso.idling:idling-concurrent:3.0.2'

//Resolve conflicts with depdencies
androidTestCompile "com.android.support:support-annotations:${supportLibVersion}"
// Resolve conflicts with depdencies
androidTestImplementation "com.android.support:support-annotations:${supportLibVersion}"

//Leakcanary
debugCompile 'com.squareup.leakcanary:leakcanary-android:1.4-beta2'
releaseCompile 'com.squareup.leakcanary:leakcanary-android-no-op:1.4-beta2'
testCompile 'com.squareup.leakcanary:leakcanary-android-no-op:1.4-beta2'
debugImplementation 'com.squareup.leakcanary:leakcanary-android:1.5.4'
releaseImplementation 'com.squareup.leakcanary:leakcanary-android-no-op:1.5.4'
testImplementation 'com.squareup.leakcanary:leakcanary-android-no-op:1.5.4'

//License dialog
compile 'de.psdev.licensesdialog:licensesdialog:1.8.1'
implementation 'de.psdev.licensesdialog:licensesdialog:1.8.1'
}
46 changes: 19 additions & 27 deletions app/src/main/java/subreddit/android/appstore/backend/scrapers/gplay/GPlayScraper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,16 @@

import android.support.annotation.NonNull;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

import io.reactivex.Observable;
import io.reactivex.functions.Function;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import subreddit.android.appstore.backend.DeadLinkException;
Expand Down Expand Up @@ -40,41 +46,26 @@ public Observable<ScrapeResult> get(@NonNull final AppInfo appToScrape) {
})
.map(s -> client.newCall(new Request.Builder().url(s).build()).execute())
.map(response -> {
if (response.code() == 404)
throw new DeadLinkException(response.request().url().toString());
if (response.code() == 404) throw new DeadLinkException(response.request().url().toString());

Collection<String> urls = new ArrayList<>();
String body = response.body().string();
int iconStart = body.indexOf("<img class=\"cover-image\"");
int iconEnd = body.indexOf("\" alt=\"Cover art\"", iconStart);
String iconUrl = body.substring(body.indexOf("lh", iconStart), iconEnd);
Timber.v("%s | icon url: %s", appToScrape, iconUrl);
Document doc = Jsoup.parse(response.body().string());

String iconUrl = doc.select("img[alt*=Cover Art]").attr("src");
// Strip size parameter we generate these
iconUrl = iconUrl.replaceAll("=(w|h)\\d+", "");

if (!iconUrl.startsWith("http://")) iconUrl = "http://" + iconUrl;
while (body.contains("<img class=\"screenshot\"")) {
int start = body.indexOf("<img class=\"screenshot\"");
int end = body.indexOf("itemprop=\"screenshot\"", start);
String screenUrl = body.substring(start, end);
int subStart = screenUrl.indexOf("lh");
int subEnd = screenUrl.indexOf("\"", subStart);
screenUrl = screenUrl.substring(subStart, subEnd);
body = body.substring(end, body.length());
Timber.v("%s | screenshot url: %s", appToScrape, screenUrl);
iconUrl = iconUrl.replaceAll("=(s|w|h)\\d+", "");

Collection<String> screenUrls = new ArrayList<>();
for (Element screenshots : doc.select("img[alt*=Screenshot Image]")) {
String screen = screenshots.attr("src");
// Strip size parameter we generate these
screenUrl = screenUrl.replaceAll("=(w|h)\\d+", "");

if (!screenUrl.startsWith("http://")) screenUrl = "http://" + screenUrl;
urls.add(screenUrl);
screen = screen.replaceAll("=(-*(w|h|s)\\d+)*", "");
screenUrls.add(screen);
}

return new GPlayResult(iconUrl, urls);
return new GPlayResult(iconUrl, screenUrls);
})
.toList()
.map(scrapeResults -> {
.map((Function<List<GPlayResult>, ScrapeResult>) scrapeResults -> {
String iconUrl = null;
Collection<String> screenshotUrls = new ArrayList<>();
for (ScrapeResult scrapeResult : scrapeResults) {
Expand All @@ -84,7 +75,8 @@ public Observable<ScrapeResult> get(@NonNull final AppInfo appToScrape) {
screenshotUrls.addAll(scrapeResult.getScreenshotUrls());
}
return new GPlayResult(iconUrl, screenshotUrls);
});
})
.toObservable();

}
}
5 changes: 3 additions & 2 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
buildscript {
repositories {
jcenter()
google()
}
dependencies {
classpath 'com.android.tools.build:gradle:2.3.3'
classpath 'com.android.tools.build:gradle:3.1.3'
classpath "io.realm:realm-gradle-plugin:3.1.2"
classpath 'me.tatarka:gradle-retrolambda:3.3.0'
classpath 'org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.4.0'
}
}
Expand All @@ -17,6 +17,7 @@ allprojects {
jcenter()
maven { url "https://jitpack.io" }
maven { url "https://oss.sonatype.org/content/repositories/snapshots" }
google()
}
}
task clean(type: Delete) {
Expand Down
4 changes: 2 additions & 2 deletions gradle/wrapper/gradle-wrapper.properties
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#Mon Jun 26 03:22:15 CEST 2017
#Fri Jul 27 16:48:48 CEST 2018
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip

0 comments on commit 0376cd4

Please sign in to comment.