diff --git a/app/src/main/assets/data.minify.json b/app/src/main/assets/data.minify.json index 4f98c89..a22a65d 100644 --- a/app/src/main/assets/data.minify.json +++ b/app/src/main/assets/data.minify.json @@ -136,7 +136,7 @@ "exceptions": [ "^https?:\\/\\/mail\\.google\\.com\\/mail\\/u\\/", "^https?:\\/\\/(?:docs|accounts)\\.google(?:\\.[a-z]{2,}){1,}", - "^https?:\\/\\/drive\\.google\\.com\\/videoplayback", + "^https?:\\/\\/([a-z0-9-\\.])*drive\\.google\\.com\\/videoplayback", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}(?:\\/upload)?\\/drive\\/", "^https?:\\/\\/news\\.google\\.com.*\\?hl=.", "^https?:\\/\\/hangouts\\.google\\.com\\/webchat.*?zx=.", @@ -240,7 +240,11 @@ "^https?:\\/\\/api\\.bilibili\\.com", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?onet\\.pl\\/[^?]*\\?.*?utm_campaign=.", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?stripe\\.com\\/[^?]+.*?&?referrer=[^/?&]*", - "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lichess\\.org\\/login.*?&?referrer=.*?" + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lichess\\.org\\/login.*?&?referrer=.*?", + "^https?:\\/\\/like.co\\/api\\/like\\/likebutton\\/[^?]+.*?&?referrer=[^/?&]*", + "^https?:\\/\\/button.like.co\\/in\\/.*?&?referrer=[^/?&]*", + "^https?:\\/\\/www\\.mma\\.go\\.kr", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?github\\.com" ] }, "adtech": { @@ -1023,7 +1027,8 @@ "impid", "bftTag", "bftRwd", - "spm" + "spm", + "_u" ] }, "tmall.com": { @@ -1501,6 +1506,71 @@ "rules": [ "intcid" ] + }, + "alibaba cloud arms": { + "urlPattern": "^https?:\\/\\/arms-retcode\\.aliyuncs\\.com", + "rules": [ + "pid", + "uid", + "tag", + "release", + "environment", + "sample", + "behavior", + "enableSPA", + "enableLinkTrace", + "page", + "begin", + "c2", + "c3", + "success", + "code", + "msg", + "api", + "traceId", + "pv_id", + "flag", + "sr", + "vp", + "ct", + "_v", + "sampling", + "dl", + "post_res" + ] + }, + "nikkei": { + "urlPattern": "^https?://(?:[a-z0-9-]+\\.)*?nikkei\\.co(?:m|\\.jp)", + "rules": [ + "adid", + "i_cid", + "n_cid", + "waad" + ] + }, + "weibo": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?weibo\\.(cn|com)", + "rules": [ + "weibo_id", + "dt_dapp" + ] + }, + "fiverr.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?fiverr\\.com", + "rules": [ + "context_referrer", + "source", + "ref_ctx_id", + "funnel" + ] + }, + "etsy.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?etsy\\.com", + "rules": [ + "click_key", + "click_sum", + "organic_search_click" + ] } } } \ No newline at end of file diff --git a/app/src/main/java/com/trianguloy/urlchecker/modules/list/ClearUrlModule.java b/app/src/main/java/com/trianguloy/urlchecker/modules/list/ClearUrlModule.java index c242e07..f0a40b8 100644 --- a/app/src/main/java/com/trianguloy/urlchecker/modules/list/ClearUrlModule.java +++ b/app/src/main/java/com/trianguloy/urlchecker/modules/list/ClearUrlModule.java @@ -137,18 +137,19 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener { whileProvider: while (providers.hasNext()) { + // evaluate each provider String provider = providers.next(); JSONObject providerData = data.getJSONObject(provider); - if (!Pattern.compile(providerData.getString("urlPattern")).matcher(cleared).find()) { + if (!matcher(providerData.getString("urlPattern"), cleared).find()) { continue; } // info - if (verbose.get()) - append(R.string.mClear_matches, provider); + if (verbose.get()) append(R.string.mClear_matches, provider); // check blocked completeProvider if (providerData.optBoolean("completeProvider", false)) { + // provider blocked append(R.string.mClear_blocked); setColor(R.color.bad); continue; @@ -159,9 +160,9 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener { JSONArray exceptions = providerData.getJSONArray("exceptions"); for (int i = 0; i < exceptions.length(); i++) { String exception = exceptions.getString(i); - if (Pattern.compile(exception).matcher(cleared).find()) { - if (verbose.get()) - append(R.string.mClear_exception); + if (matcher(exception, cleared).find()) { + // exception matches, ignore provider + if (verbose.get()) append(R.string.mClear_exception, exception); continue whileProvider; } } @@ -174,42 +175,49 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener { String redirection = redirections.getString(i); Matcher matcher = Pattern.compile(redirection).matcher(cleared); if (matcher.find() && matcher.groupCount() >= 1) { + // redirection found if (providerData.optBoolean("forceRedirection", false)) { // maybe do something special? append(R.string.mClear_forcedRedirection); } else { append(R.string.mClear_redirection); } - cleared = decodeURIComponent(matcher.group(1)); + if (verbose.get()) details(redirection); + cleared = decodeURIComponent(matcher.group(1)); // can't be null, checked in the if setColor(R.color.warning); continue whileProvider; } } } - // apply rules - if (providerData.has("rules")) { - JSONArray rules = providerData.getJSONArray("rules"); - for (int i = 0; i < rules.length(); i++) { - String rule = "(?:&|[/?#&])(?:" + rules.getString(i) + "=[^&]*)"; - Matcher matcher = Pattern.compile(rule).matcher(cleared); - if (matcher.find()) { - cleared = matcher.replaceAll(""); - append(R.string.mClear_rule); - setColor(R.color.warning); - } - } - } - // apply rawRules if (providerData.has("rawRules")) { JSONArray rawRules = providerData.getJSONArray("rawRules"); for (int i = 0; i < rawRules.length(); i++) { String rawRule = rawRules.getString(i); - Matcher matcher = Pattern.compile(rawRule).matcher(cleared); + Matcher matcher = matcher(rawRule, cleared); if (matcher.find()) { + // rawrule matches, apply cleared = matcher.replaceAll(""); append(R.string.mClear_rawRule); + if (verbose.get()) details(rawRule); + setColor(R.color.warning); + } + } + } + + // apply rules + if (providerData.has("rules")) { + JSONArray rules = providerData.getJSONArray("rules"); + for (int i = 0; i < rules.length(); i++) { + String rule = "([?&#])" + rules.getString(i) + "=[^&#]*"; + Matcher matcher = matcher(rule, cleared); + while (matcher.find()) { + // rule applies + cleared = matcher.replaceFirst("$1"); + matcher.reset(cleared); + append(R.string.mClear_rule); + if (verbose.get()) details(rules.getString(i)); setColor(R.color.warning); } } @@ -219,15 +227,31 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener { if (!allowReferral.get() && providerData.has("referralMarketing")) { JSONArray referrals = providerData.getJSONArray("referralMarketing"); for (int i = 0; i < referrals.length(); i++) { - String referral = "(?:&|[/?#&])(?:" + referrals.getString(i) + "=[^&]*)"; - Matcher matcher = Pattern.compile(referral).matcher(cleared); - if (matcher.find()) { - cleared = matcher.replaceAll(""); + String referral = "([?&#])" + referrals.getString(i) + "=[^&#]*"; + Matcher matcher = matcher(referral, cleared); + while (matcher.find()) { + // raw rule applies + cleared = matcher.replaceFirst("$1"); + matcher.reset(cleared); append(R.string.mClear_referral); + if (verbose.get()) details(referrals.getString(i)); setColor(R.color.warning); } } } + + // fix empty elements + cleared = cleared + .replaceAll("\\?&+", "?") + .replaceAll("\\?#", "#") + .replaceAll("\\?$", "") + .replaceAll("&&+", "&") + .replaceAll("&#", "#") + .replaceAll("&$", "") + .replaceAll("#&+", "#") + .replaceAll("#$", "") + ; + } } catch (JSONException | UnsupportedEncodingException e) { e.printStackTrace(); @@ -237,6 +261,7 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener { // url changed, enable button if (!cleared.equals(url)) { fix.setEnabled(true); + if (verbose.get()) info.append("\n\n -> " + cleared); } // nothing found @@ -267,6 +292,17 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener { return result.toString(); } + /** + * Matches cas insnsitive regexp into an input + * + * @param regexp regexp to use + * @param input input to use + * @return the matcher object + */ + private static Matcher matcher(String regexp, String input) { + return Pattern.compile(regexp, Pattern.CASE_INSENSITIVE).matcher(input); + } + /** * Utility to append a line to the info textview, manages linebreaks */ @@ -275,6 +311,13 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener { info.append(getActivity().getString(line, formatArgs)); } + /** + * utility to append data to the info textview, after calling append + */ + private void details(String data) { + info.append(": " + data); + } + /** * Utility to set the info background color. Manages color importance */ @@ -285,7 +328,7 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener { } /** - * Reads a JSON file and returns its content + * Reads a file and returns its content * From https://www.bezkoder.com/java-android-read-json-file-assets-gson/ */ static String getJsonFromAssets(Context context, String fileName) throws IOException { diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 8506eda..de9ea9f 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -91,11 +91,11 @@ Clear URL This module removes tracking, referrer and other useless parameters from the url. It also allows for common offline url redirections. - Uses the Clear URL database (version March 30, 2021) from https://docs.clearurls.xyz/latest/specs/rules/ + Uses the Clear URL database (version February 11, 2022) from https://docs.clearurls.xyz/latest/specs/rules/ Apply Matches %s - BLOCKED! - - ignored due to exception + - ignored due to exception %s - useless parameter found - useless content found - referral found diff --git a/build.gradle b/build.gradle index 023e3b6..a72e504 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:7.0.3' + classpath 'com.android.tools.build:gradle:7.0.4' // NOTE: Do not place your application dependencies here; they belong