diff --git a/app/src/main/assets/data.minify.json b/app/src/main/assets/data.minify.json
index 4f98c89..a22a65d 100644
--- a/app/src/main/assets/data.minify.json
+++ b/app/src/main/assets/data.minify.json
@@ -136,7 +136,7 @@
"exceptions": [
"^https?:\\/\\/mail\\.google\\.com\\/mail\\/u\\/",
"^https?:\\/\\/(?:docs|accounts)\\.google(?:\\.[a-z]{2,}){1,}",
- "^https?:\\/\\/drive\\.google\\.com\\/videoplayback",
+ "^https?:\\/\\/([a-z0-9-\\.])*drive\\.google\\.com\\/videoplayback",
"^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}(?:\\/upload)?\\/drive\\/",
"^https?:\\/\\/news\\.google\\.com.*\\?hl=.",
"^https?:\\/\\/hangouts\\.google\\.com\\/webchat.*?zx=.",
@@ -240,7 +240,11 @@
"^https?:\\/\\/api\\.bilibili\\.com",
"^https?:\\/\\/(?:[a-z0-9-]+\\.)*?onet\\.pl\\/[^?]*\\?.*?utm_campaign=.",
"^https?:\\/\\/(?:[a-z0-9-]+\\.)*?stripe\\.com\\/[^?]+.*?&?referrer=[^/?&]*",
- "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lichess\\.org\\/login.*?&?referrer=.*?"
+ "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lichess\\.org\\/login.*?&?referrer=.*?",
+ "^https?:\\/\\/like.co\\/api\\/like\\/likebutton\\/[^?]+.*?&?referrer=[^/?&]*",
+ "^https?:\\/\\/button.like.co\\/in\\/.*?&?referrer=[^/?&]*",
+ "^https?:\\/\\/www\\.mma\\.go\\.kr",
+ "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?github\\.com"
]
},
"adtech": {
@@ -1023,7 +1027,8 @@
"impid",
"bftTag",
"bftRwd",
- "spm"
+ "spm",
+ "_u"
]
},
"tmall.com": {
@@ -1501,6 +1506,71 @@
"rules": [
"intcid"
]
+ },
+ "alibaba cloud arms": {
+ "urlPattern": "^https?:\\/\\/arms-retcode\\.aliyuncs\\.com",
+ "rules": [
+ "pid",
+ "uid",
+ "tag",
+ "release",
+ "environment",
+ "sample",
+ "behavior",
+ "enableSPA",
+ "enableLinkTrace",
+ "page",
+ "begin",
+ "c2",
+ "c3",
+ "success",
+ "code",
+ "msg",
+ "api",
+ "traceId",
+ "pv_id",
+ "flag",
+ "sr",
+ "vp",
+ "ct",
+ "_v",
+ "sampling",
+ "dl",
+ "post_res"
+ ]
+ },
+ "nikkei": {
+ "urlPattern": "^https?://(?:[a-z0-9-]+\\.)*?nikkei\\.co(?:m|\\.jp)",
+ "rules": [
+ "adid",
+ "i_cid",
+ "n_cid",
+ "waad"
+ ]
+ },
+ "weibo": {
+ "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?weibo\\.(cn|com)",
+ "rules": [
+ "weibo_id",
+ "dt_dapp"
+ ]
+ },
+ "fiverr.com": {
+ "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?fiverr\\.com",
+ "rules": [
+ "context_referrer",
+ "source",
+ "ref_ctx_id",
+ "funnel"
+ ]
+ },
+ "etsy.com": {
+ "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?etsy\\.com",
+ "rules": [
+ "click_key",
+ "click_sum",
+ "organic_search_click"
+ ]
}
}
}
\ No newline at end of file
diff --git a/app/src/main/java/com/trianguloy/urlchecker/modules/list/ClearUrlModule.java b/app/src/main/java/com/trianguloy/urlchecker/modules/list/ClearUrlModule.java
index c242e07..f0a40b8 100644
--- a/app/src/main/java/com/trianguloy/urlchecker/modules/list/ClearUrlModule.java
+++ b/app/src/main/java/com/trianguloy/urlchecker/modules/list/ClearUrlModule.java
@@ -137,18 +137,19 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
whileProvider:
while (providers.hasNext()) {
+ // evaluate each provider
String provider = providers.next();
JSONObject providerData = data.getJSONObject(provider);
- if (!Pattern.compile(providerData.getString("urlPattern")).matcher(cleared).find()) {
+ if (!matcher(providerData.getString("urlPattern"), cleared).find()) {
continue;
}
// info
- if (verbose.get())
- append(R.string.mClear_matches, provider);
+ if (verbose.get()) append(R.string.mClear_matches, provider);
// check blocked completeProvider
if (providerData.optBoolean("completeProvider", false)) {
+ // provider blocked
append(R.string.mClear_blocked);
setColor(R.color.bad);
continue;
@@ -159,9 +160,9 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
JSONArray exceptions = providerData.getJSONArray("exceptions");
for (int i = 0; i < exceptions.length(); i++) {
String exception = exceptions.getString(i);
- if (Pattern.compile(exception).matcher(cleared).find()) {
- if (verbose.get())
- append(R.string.mClear_exception);
+ if (matcher(exception, cleared).find()) {
+ // exception matches, ignore provider
+ if (verbose.get()) append(R.string.mClear_exception, exception);
continue whileProvider;
}
}
@@ -174,42 +175,49 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
String redirection = redirections.getString(i);
Matcher matcher = Pattern.compile(redirection).matcher(cleared);
if (matcher.find() && matcher.groupCount() >= 1) {
+ // redirection found
if (providerData.optBoolean("forceRedirection", false)) {
// maybe do something special?
append(R.string.mClear_forcedRedirection);
} else {
append(R.string.mClear_redirection);
}
- cleared = decodeURIComponent(matcher.group(1));
+ if (verbose.get()) details(redirection);
+ cleared = decodeURIComponent(matcher.group(1)); // can't be null, checked in the if
setColor(R.color.warning);
continue whileProvider;
}
}
}
- // apply rules
- if (providerData.has("rules")) {
- JSONArray rules = providerData.getJSONArray("rules");
- for (int i = 0; i < rules.length(); i++) {
- String rule = "(?:&|[/?#&])(?:" + rules.getString(i) + "=[^&]*)";
- Matcher matcher = Pattern.compile(rule).matcher(cleared);
- if (matcher.find()) {
- cleared = matcher.replaceAll("");
- append(R.string.mClear_rule);
- setColor(R.color.warning);
- }
- }
- }
-
// apply rawRules
if (providerData.has("rawRules")) {
JSONArray rawRules = providerData.getJSONArray("rawRules");
for (int i = 0; i < rawRules.length(); i++) {
String rawRule = rawRules.getString(i);
- Matcher matcher = Pattern.compile(rawRule).matcher(cleared);
+ Matcher matcher = matcher(rawRule, cleared);
if (matcher.find()) {
+ // rawrule matches, apply
cleared = matcher.replaceAll("");
append(R.string.mClear_rawRule);
+ if (verbose.get()) details(rawRule);
+ setColor(R.color.warning);
+ }
+ }
+ }
+
+ // apply rules
+ if (providerData.has("rules")) {
+ JSONArray rules = providerData.getJSONArray("rules");
+ for (int i = 0; i < rules.length(); i++) {
+ String rule = "([?])" + rules.getString(i) + "=[^]*";
+ Matcher matcher = matcher(rule, cleared);
+ while (matcher.find()) {
+ // rule applies
+ cleared = matcher.replaceFirst("$1");
+ matcher.reset(cleared);
+ append(R.string.mClear_rule);
+ if (verbose.get()) details(rules.getString(i));
setColor(R.color.warning);
}
}
@@ -219,15 +227,31 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
if (!allowReferral.get() && providerData.has("referralMarketing")) {
JSONArray referrals = providerData.getJSONArray("referralMarketing");
for (int i = 0; i < referrals.length(); i++) {
- String referral = "(?:&|[/?#&])(?:" + referrals.getString(i) + "=[^&]*)";
- Matcher matcher = Pattern.compile(referral).matcher(cleared);
- if (matcher.find()) {
- cleared = matcher.replaceAll("");
+ String referral = "([?])" + referrals.getString(i) + "=[^]*";
+ Matcher matcher = matcher(referral, cleared);
+ while (matcher.find()) {
+ // raw rule applies
+ cleared = matcher.replaceFirst("$1");
+ matcher.reset(cleared);
append(R.string.mClear_referral);
+ if (verbose.get()) details(referrals.getString(i));
setColor(R.color.warning);
}
}
}
+
+ // fix empty elements
+ cleared = cleared
+ .replaceAll("\\?&+", "?")
+ .replaceAll("\\?#", "#")
+ .replaceAll("\\?$", "")
+ .replaceAll("&&+", "&")
+ .replaceAll("", "#")
+ .replaceAll("&$", "")
+ .replaceAll("#&+", "#")
+ .replaceAll("#$", "")
+ ;
+
}
} catch (JSONException | UnsupportedEncodingException e) {
e.printStackTrace();
@@ -237,6 +261,7 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
// url changed, enable button
if (!cleared.equals(url)) {
fix.setEnabled(true);
+ if (verbose.get()) info.append("\n\n -> " + cleared);
}
// nothing found
@@ -267,6 +292,17 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
return result.toString();
}
+ /**
+ * Matches cas insnsitive regexp into an input
+ *
+ * @param regexp regexp to use
+ * @param input input to use
+ * @return the matcher object
+ */
+ private static Matcher matcher(String regexp, String input) {
+ return Pattern.compile(regexp, Pattern.CASE_INSENSITIVE).matcher(input);
+ }
+
/**
* Utility to append a line to the info textview, manages linebreaks
*/
@@ -275,6 +311,13 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
info.append(getActivity().getString(line, formatArgs));
}
+ /**
+ * utility to append data to the info textview, after calling append
+ */
+ private void details(String data) {
+ info.append(": " + data);
+ }
+
/**
* Utility to set the info background color. Manages color importance
*/
@@ -285,7 +328,7 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
}
/**
- * Reads a JSON file and returns its content
+ * Reads a file and returns its content
* From https://www.bezkoder.com/java-android-read-json-file-assets-gson/
*/
static String getJsonFromAssets(Context context, String fileName) throws IOException {
diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml
index 8506eda..de9ea9f 100644
--- a/app/src/main/res/values/strings.xml
+++ b/app/src/main/res/values/strings.xml
@@ -91,11 +91,11 @@
Clear URL
This module removes tracking, referrer and other useless parameters from the url. It also allows for common offline url redirections.
- Uses the Clear URL database (version March 30, 2021) from https://docs.clearurls.xyz/latest/specs/rules/
+ Uses the Clear URL database (version February 11, 2022) from https://docs.clearurls.xyz/latest/specs/rules/
Apply
Matches %s
- BLOCKED!
- - ignored due to exception
+ - ignored due to exception %s
- useless parameter found
- useless content found
- referral found
diff --git a/build.gradle b/build.gradle
index 023e3b6..a72e504 100644
--- a/build.gradle
+++ b/build.gradle
@@ -7,7 +7,7 @@ buildscript {
mavenCentral()
}
dependencies {
- classpath 'com.android.tools.build:gradle:7.0.3'
+ classpath 'com.android.tools.build:gradle:7.0.4'
// NOTE: Do not place your application dependencies here; they belong