several fixes and improvements for the clear url module

fixes #4

app/src/main/assets/data.minify.json

@@ -136,7 +136,7 @@
       "exceptions": [
         "^https?:\\/\\/mail\\.google\\.com\\/mail\\/u\\/",
         "^https?:\\/\\/(?:docs|accounts)\\.google(?:\\.[a-z]{2,}){1,}",
-        "^https?:\\/\\/drive\\.google\\.com\\/videoplayback",
+        "^https?:\\/\\/([a-z0-9-\\.])*drive\\.google\\.com\\/videoplayback",
         "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}(?:\\/upload)?\\/drive\\/",
         "^https?:\\/\\/news\\.google\\.com.*\\?hl=.",
         "^https?:\\/\\/hangouts\\.google\\.com\\/webchat.*?zx=.",
@@ -240,7 +240,11 @@
         "^https?:\\/\\/api\\.bilibili\\.com",
         "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?onet\\.pl\\/[^?]*\\?.*?utm_campaign=.",
         "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?stripe\\.com\\/[^?]+.*?&?referrer=[^/?&]*",
-        "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lichess\\.org\\/login.*?&?referrer=.*?"
+        "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lichess\\.org\\/login.*?&?referrer=.*?",
+        "^https?:\\/\\/like.co\\/api\\/like\\/likebutton\\/[^?]+.*?&?referrer=[^/?&]*",
+        "^https?:\\/\\/button.like.co\\/in\\/.*?&?referrer=[^/?&]*",
+        "^https?:\\/\\/www\\.mma\\.go\\.kr",
+        "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?github\\.com"
       ]
     },
     "adtech": {
@@ -1023,7 +1027,8 @@
         "impid",
         "bftTag",
         "bftRwd",
-        "spm"
+        "spm",
+        "_u"
       ]
     },
     "tmall.com": {
@@ -1501,6 +1506,71 @@
       "rules": [
         "intcid"
       ]
+    },
+    "alibaba cloud arms": {
+      "urlPattern": "^https?:\\/\\/arms-retcode\\.aliyuncs\\.com",
+      "rules": [
+        "pid",
+        "uid",
+        "tag",
+        "release",
+        "environment",
+        "sample",
+        "behavior",
+        "enableSPA",
+        "enableLinkTrace",
+        "page",
+        "begin",
+        "c2",
+        "c3",
+        "success",
+        "code",
+        "msg",
+        "api",
+        "traceId",
+        "pv_id",
+        "flag",
+        "sr",
+        "vp",
+        "ct",
+        "_v",
+        "sampling",
+        "dl",
+        "post_res"
+      ]
+    },
+    "nikkei": {
+      "urlPattern": "^https?://(?:[a-z0-9-]+\\.)*?nikkei\\.co(?:m|\\.jp)",
+      "rules": [
+        "adid",
+        "i_cid",
+        "n_cid",
+        "waad"
+      ]
+    },
+    "weibo": {
+      "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?weibo\\.(cn|com)",
+      "rules": [
+        "weibo_id",
+        "dt_dapp"
+      ]
+    },
+    "fiverr.com": {
+      "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?fiverr\\.com",
+      "rules": [
+        "context_referrer",
+        "source",
+        "ref_ctx_id",
+        "funnel"
+      ]
+    },
+    "etsy.com": {
+      "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?etsy\\.com",
+      "rules": [
+        "click_key",
+        "click_sum",
+        "organic_search_click"
+      ]
     }
   }
 }

app/src/main/java/com/trianguloy/urlchecker/modules/list/ClearUrlModule.java

@@ -137,18 +137,19 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
 
             whileProvider:
             while (providers.hasNext()) {
+                // evaluate each provider
                 String provider = providers.next();
                 JSONObject providerData = data.getJSONObject(provider);
-                if (!Pattern.compile(providerData.getString("urlPattern")).matcher(cleared).find()) {
+                if (!matcher(providerData.getString("urlPattern"), cleared).find()) {
                     continue;
                 }
 
                 // info
-                if (verbose.get())
+                if (verbose.get()) append(R.string.mClear_matches, provider);
-                    append(R.string.mClear_matches, provider);
 
                 // check blocked completeProvider
                 if (providerData.optBoolean("completeProvider", false)) {
+                    // provider blocked
                     append(R.string.mClear_blocked);
                     setColor(R.color.bad);
                     continue;
@@ -159,9 +160,9 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
                     JSONArray exceptions = providerData.getJSONArray("exceptions");
                     for (int i = 0; i < exceptions.length(); i++) {
                         String exception = exceptions.getString(i);
-                        if (Pattern.compile(exception).matcher(cleared).find()) {
+                        if (matcher(exception, cleared).find()) {
-                            if (verbose.get())
+                            // exception matches, ignore provider
-                                append(R.string.mClear_exception);
+                            if (verbose.get()) append(R.string.mClear_exception, exception);
                             continue whileProvider;
                         }
                     }
@@ -174,42 +175,49 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
                         String redirection = redirections.getString(i);
                         Matcher matcher = Pattern.compile(redirection).matcher(cleared);
                         if (matcher.find() && matcher.groupCount() >= 1) {
+                            // redirection found
                             if (providerData.optBoolean("forceRedirection", false)) {
                                 // maybe do something special?
                                 append(R.string.mClear_forcedRedirection);
                             } else {
                                 append(R.string.mClear_redirection);
                             }
-                            cleared = decodeURIComponent(matcher.group(1));
+                            if (verbose.get()) details(redirection);
+                            cleared = decodeURIComponent(matcher.group(1)); // can't be null, checked in the if
                             setColor(R.color.warning);
                             continue whileProvider;
                         }
                     }
                 }
 
-                // apply rules
-                if (providerData.has("rules")) {
-                    JSONArray rules = providerData.getJSONArray("rules");
-                    for (int i = 0; i < rules.length(); i++) {
-                        String rule = "(?:&amp;|[/?#&])(?:" + rules.getString(i) + "=[^&]*)";
-                        Matcher matcher = Pattern.compile(rule).matcher(cleared);
-                        if (matcher.find()) {
-                            cleared = matcher.replaceAll("");
-                            append(R.string.mClear_rule);
-                            setColor(R.color.warning);
-                        }
-                    }
-                }
-
                 // apply rawRules
                 if (providerData.has("rawRules")) {
                     JSONArray rawRules = providerData.getJSONArray("rawRules");
                     for (int i = 0; i < rawRules.length(); i++) {
                         String rawRule = rawRules.getString(i);
-                        Matcher matcher = Pattern.compile(rawRule).matcher(cleared);
+                        Matcher matcher = matcher(rawRule, cleared);
                         if (matcher.find()) {
+                            // rawrule matches, apply
                             cleared = matcher.replaceAll("");
                             append(R.string.mClear_rawRule);
+                            if (verbose.get()) details(rawRule);
+                            setColor(R.color.warning);
+                        }
+                    }
+                }
+
+                // apply rules
+                if (providerData.has("rules")) {
+                    JSONArray rules = providerData.getJSONArray("rules");
+                    for (int i = 0; i < rules.length(); i++) {
+                        String rule = "([?&#])" + rules.getString(i) + "=[^&#]*";
+                        Matcher matcher = matcher(rule, cleared);
+                        while (matcher.find()) {
+                            // rule applies
+                            cleared = matcher.replaceFirst("$1");
+                            matcher.reset(cleared);
+                            append(R.string.mClear_rule);
+                            if (verbose.get()) details(rules.getString(i));
                             setColor(R.color.warning);
                         }
                     }
@@ -219,15 +227,31 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
                 if (!allowReferral.get() && providerData.has("referralMarketing")) {
                     JSONArray referrals = providerData.getJSONArray("referralMarketing");
                     for (int i = 0; i < referrals.length(); i++) {
-                        String referral = "(?:&amp;|[/?#&])(?:" + referrals.getString(i) + "=[^&]*)";
+                        String referral = "([?&#])" + referrals.getString(i) + "=[^&#]*";
-                        Matcher matcher = Pattern.compile(referral).matcher(cleared);
+                        Matcher matcher = matcher(referral, cleared);
-                        if (matcher.find()) {
+                        while (matcher.find()) {
-                            cleared = matcher.replaceAll("");
+                            // raw rule applies
+                            cleared = matcher.replaceFirst("$1");
+                            matcher.reset(cleared);
                             append(R.string.mClear_referral);
+                            if (verbose.get()) details(referrals.getString(i));
                             setColor(R.color.warning);
                         }
                     }
                 }
+
+                // fix empty elements
+                cleared = cleared
+                        .replaceAll("\\?&+", "?")
+                        .replaceAll("\\?#", "#")
+                        .replaceAll("\\?$", "")
+                        .replaceAll("&&+", "&")
+                        .replaceAll("&#", "#")
+                        .replaceAll("&$", "")
+                        .replaceAll("#&+", "#")
+                        .replaceAll("#$", "")
+                ;
+
             }
         } catch (JSONException | UnsupportedEncodingException e) {
             e.printStackTrace();
@@ -237,6 +261,7 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
         // url changed, enable button
         if (!cleared.equals(url)) {
             fix.setEnabled(true);
+            if (verbose.get()) info.append("\n\n -> " + cleared);
         }
 
         // nothing found
@@ -267,6 +292,17 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
         return result.toString();
     }
 
+    /**
+     * Matches cas insnsitive regexp into an input
+     *
+     * @param regexp regexp to use
+     * @param input  input to use
+     * @return the matcher object
+     */
+    private static Matcher matcher(String regexp, String input) {
+        return Pattern.compile(regexp, Pattern.CASE_INSENSITIVE).matcher(input);
+    }
+
     /**
      * Utility to append a line to the info textview, manages linebreaks
      */
@@ -275,6 +311,13 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
         info.append(getActivity().getString(line, formatArgs));
     }
 
+    /**
+     * utility to append data to the info textview, after calling append
+     */
+    private void details(String data) {
+        info.append(": " + data);
+    }
+
     /**
      * Utility to set the info background color. Manages color importance
      */
@@ -285,7 +328,7 @@ class ClearUrlDialog extends AModuleDialog implements View.OnClickListener {
     }
 
     /**
-     * Reads a JSON file and returns its content
+     * Reads a file and returns its content
      * From https://www.bezkoder.com/java-android-read-json-file-assets-gson/
      */
     static String getJsonFromAssets(Context context, String fileName) throws IOException {

app/src/main/res/values/strings.xml

@@ -91,11 +91,11 @@
 
     <string name="mClear_name">Clear URL</string>
     <string name="mClear_desc">This module removes tracking, referrer and other useless parameters from the url. It also allows for common offline url redirections.</string>
-    <string name="mClear_tm">Uses the Clear URL database (version March 30, 2021) from https://docs.clearurls.xyz/latest/specs/rules/</string>
+    <string name="mClear_tm">Uses the Clear URL database (version February 11, 2022) from https://docs.clearurls.xyz/latest/specs/rules/</string>
     <string name="mClear_clear">Apply</string>
     <string name="mClear_matches">Matches %s</string>
     <string name="mClear_blocked">- BLOCKED!</string>
-    <string name="mClear_exception">- ignored due to exception</string>
+    <string name="mClear_exception">- ignored due to exception %s</string>
     <string name="mClear_rule">- useless parameter found</string>
     <string name="mClear_rawRule">- useless content found</string>
     <string name="mClear_referral">- referral found</string>

build.gradle

@@ -7,7 +7,7 @@ buildscript {
         mavenCentral()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:7.0.3'
+        classpath 'com.android.tools.build:gradle:7.0.4'
         
 
         // NOTE: Do not place your application dependencies here; they belong