diff --git a/AnkiDroid/kotlinMigration.gradle b/AnkiDroid/kotlinMigration.gradle index ec767f3266..a2e505b236 100644 --- a/AnkiDroid/kotlinMigration.gradle +++ b/AnkiDroid/kotlinMigration.gradle @@ -43,7 +43,7 @@ permission notice: // Example of class name: "/com/ichi2/anki/UIUtils.kt" // Ensure that it starts with '/' (slash) def source = Source.MAIN -def className = "/com/ichi2/libanki/Media.kt" +def className = "" enum Source { MAIN("/src/main/java"), diff --git a/AnkiDroid/src/androidTest/java/com/ichi2/anki/tests/libanki/MediaTest.kt b/AnkiDroid/src/androidTest/java/com/ichi2/anki/tests/libanki/MediaTest.kt index 56081c8b39..30c5f6bedf 100644 --- a/AnkiDroid/src/androidTest/java/com/ichi2/anki/tests/libanki/MediaTest.kt +++ b/AnkiDroid/src/androidTest/java/com/ichi2/anki/tests/libanki/MediaTest.kt @@ -104,43 +104,43 @@ class MediaTest : InstrumentedTest() { val mid = mTestCol!!.models.getModels().entries.iterator().next().key var expected: List = emptyList() - var actual = mTestCol!!.media.filesInStr(mid, "aoeu") + var actual = mTestCol!!.media.filesInStr(mid, "aoeu").toMutableList() actual.retainAll(expected) assertEquals(expected.size, actual.size) expected = listOf("foo.jpg") - actual = mTestCol!!.media.filesInStr(mid, "aoeuao") + actual = mTestCol!!.media.filesInStr(mid, "aoeuao").toMutableList() actual.retainAll(expected) assertEquals(expected.size, actual.size) expected = listOf("foo.jpg", "bar.jpg") - actual = mTestCol!!.media.filesInStr(mid, "aoeuao") + actual = mTestCol!!.media.filesInStr(mid, "aoeuao").toMutableList() actual.retainAll(expected) assertEquals(expected.size, actual.size) expected = listOf("foo.jpg") - actual = mTestCol!!.media.filesInStr(mid, "aoeuao") + actual = mTestCol!!.media.filesInStr(mid, "aoeuao").toMutableList() actual.retainAll(expected) assertEquals(expected.size, actual.size) expected = listOf("one", "two") - actual = mTestCol!!.media.filesInStr(mid, "") + actual = mTestCol!!.media.filesInStr(mid, "").toMutableList() actual.retainAll(expected) assertEquals(expected.size, actual.size) expected = listOf("foo.jpg") - actual = mTestCol!!.media.filesInStr(mid, "aoeuao") + actual = mTestCol!!.media.filesInStr(mid, "aoeuao").toMutableList() actual.retainAll(expected) assertEquals(expected.size, actual.size) expected = listOf("foo.jpg", "fo") actual = - mTestCol!!.media.filesInStr(mid, "aoeuao") + mTestCol!!.media.filesInStr(mid, "aoeuao").toMutableList() actual.retainAll(expected) assertEquals(expected.size, actual.size) expected = listOf("foo.mp3") - actual = mTestCol!!.media.filesInStr(mid, "aou[sound:foo.mp3]aou") + actual = mTestCol!!.media.filesInStr(mid, "aou[sound:foo.mp3]aou").toMutableList() actual.retainAll(expected) assertEquals(expected.size, actual.size) @@ -183,11 +183,11 @@ class MediaTest : InstrumentedTest() { // check media val ret = mTestCol!!.media.check() var expected = listOf("fake2.png") - var actual = ret[0] + var actual = ret[0].toMutableList() actual.retainAll(expected) assertEquals(expected.size, actual.size) expected = listOf("foo.jpg") - actual = ret[1] + actual = ret[1].toMutableList() actual.retainAll(expected) assertEquals(expected.size, actual.size) } @@ -209,11 +209,11 @@ class MediaTest : InstrumentedTest() { } private fun added(d: Collection?): List { - return d!!.media.db.queryStringList("select fname from media where csum is not null") + return d!!.media.db!!.queryStringList("select fname from media where csum is not null") } private fun removed(d: Collection?): List { - return d!!.media.db.queryStringList("select fname from media where csum is null") + return d!!.media.db!!.queryStringList("select fname from media where csum is null") } @Test diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/DB.kt b/AnkiDroid/src/main/java/com/ichi2/libanki/DB.kt index 7e8d8f524d..d9ada4157d 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/DB.kt +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/DB.kt @@ -186,7 +186,7 @@ class DB(db: SupportSQLiteDatabase) { return results } - fun execute(@Language("SQL") sql: String, vararg `object`: Any) { + fun execute(@Language("SQL") sql: String, vararg `object`: Any?) { val s = sql.trim { it <= ' ' }.lowercase() // mark modified? for (mo in MOD_SQLS) { @@ -230,7 +230,7 @@ class DB(db: SupportSQLiteDatabase) { return database.insert(table, SQLiteDatabase.CONFLICT_NONE, values) } - fun executeMany(@Language("SQL") sql: String, list: List>) { + fun executeMany(@Language("SQL") sql: String, list: List>) { mod = true if (BuildConfig.DEBUG) { if (list.size <= 1) { @@ -245,7 +245,7 @@ class DB(db: SupportSQLiteDatabase) { /** Use this executeMany version with external transaction management */ @KotlinCleanup("Use forEach") - fun executeManyNoTransaction(@Language("SQL") sql: String, list: List>) { + fun executeManyNoTransaction(@Language("SQL") sql: String, list: List>) { mod = true for (o in list) { database.execSQL(sql, o) diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/Media.kt b/AnkiDroid/src/main/java/com/ichi2/libanki/Media.kt index 886368f954..24b6ac3625 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/Media.kt +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/Media.kt @@ -15,289 +15,187 @@ * this program. If not, see . * ****************************************************************************************/ -package com.ichi2.libanki; +package com.ichi2.libanki -import android.database.Cursor; -import android.database.SQLException; -import android.net.Uri; -import android.text.TextUtils; - -import android.util.Pair; - -import com.ichi2.anki.CrashReportService; -import com.ichi2.libanki.exception.EmptyMediaException; -import com.ichi2.libanki.template.TemplateFilters; -import com.ichi2.utils.Assert; - -import com.ichi2.utils.ExceptionUtil; -import com.ichi2.utils.HashUtil; -import com.ichi2.utils.JSONArray; -import com.ichi2.utils.JSONObject; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; - -import androidx.annotation.NonNull; -import timber.log.Timber; - -import static java.lang.Math.min; +import android.database.SQLException +import android.net.Uri +import android.text.TextUtils +import android.util.Pair +import com.ichi2.anki.CrashReportService +import com.ichi2.libanki.exception.EmptyMediaException +import com.ichi2.libanki.template.TemplateFilters +import com.ichi2.utils.* +import com.ichi2.utils.HashUtil.HashMapInit +import timber.log.Timber +import java.io.* +import java.util.* +import java.util.regex.Matcher +import java.util.regex.Pattern +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream +import kotlin.math.min /** * Media manager - handles the addition and removal of media files from the media directory (collection.media) and * maintains the media database (collection.media.ad.db2) which is used to determine the state of files for syncing. * Note that the media database has an additional prefix for AnkiDroid (.ad) to avoid any potential issues caused by * users copying the file to the desktop client and vice versa. - *

+ * + * * Unlike the python version of this module, we do not (and cannot) modify the current working directory (CWD) before * performing operations on media files. In python, the CWD is changed to the media directory, allowing it to easily * refer to the files in the media directory by name only. In Java, we must be cautious about when to specify the full * path to the file and when we need to use the filename only. In general, when we refer to a file on disk (i.e., - * creating a new File() object), we must include the full path. Use the dir() method to make this step easier.
+ * creating a new File() object), we must include the full path. Use the dir() method to make this step easier. + * * E.g: new File(dir(), "filename.jpg") */ -@SuppressWarnings({"PMD.AvoidThrowingRawExceptionTypes","PMD.AvoidReassigningParameters", - "PMD.NPathComplexity","PMD.MethodNamingConventions","PMD.ExcessiveMethodLength","PMD.OneDeclarationPerLine", - "PMD.SwitchStmtsShouldHaveDefault","PMD.EmptyIfStmt","PMD.SimplifyBooleanReturns","PMD.CollapsibleIfStatements"}) -public class Media { - - // Upstream illegal chars defined on disallowed_char() - // in https://github.com/ankitects/anki/blob/main/rslib/src/media/files.rs - private static final Pattern fIllegalCharReg = Pattern.compile("[\\[\\]><:\"/?*^\\\\|\\x00\\r\\n]"); - private static final Pattern fRemotePattern = Pattern.compile("(https?|ftp)://"); - - /* - * A note about the regular expressions below: the python code uses named groups for the image and sound patterns. - * Our version of Java doesn't support named groups, so we must use indexes instead. In the expressions below, the - * group names (e.g., ?P) have been stripped and a comment placed above indicating the index of the group - * name in the original. Refer to these indexes whenever the python code makes use of a named group. - */ +@KotlinCleanup("IDE Lint") +open class Media(private val col: Collection, server: Boolean) { + private var mDir: String? /** - * Group 1 = Contents of [sound:] tag
- * Group 2 = "fname" + * Used by unit tests only. */ - // Regexes defined on https://github.com/ankitects/anki/blob/b403f20cae8fcdd7c3ff4c8d21766998e8efaba0/pylib/anki/media.py#L34-L45 - private static final Pattern fSoundRegexps = Pattern.compile("(?i)(\\[sound:([^]]+)])"); + @KotlinCleanup("non-null + exception if used after .close()") + var db: DB? = null + private set - // src element quoted case - /** - * Group 1 = Contents of |

+ * + * * Unlike the python version of this method, we don't read the file into memory as a string. All our operations are * done on streams opened on the file, so there is no second parameter for the string object here. */ - private String writeData(File oFile) throws IOException { + @Throws(IOException::class) + private fun writeData(oFile: File): String { // get the file name - String fname = oFile.getName(); + var fname = oFile.name // make sure we write it in NFC form and return an NFC-encoded reference - fname = Utils.nfcNormalized(fname); + fname = Utils.nfcNormalized(fname) // ensure it's a valid filename - String base = cleanFilename(fname); - String[] split = Utils.splitFilename(base); - String root = split[0]; - String ext = split[1]; + val base = cleanFilename(fname) + val split = Utils.splitFilename(base) + var root = split[0] + val ext = split[1] // find the first available name - String csum = Utils.fileChecksum(oFile); + val csum = Utils.fileChecksum(oFile) while (true) { - fname = root + ext; - File path = new File(dir(), fname); + fname = root + ext + val path = File(dir(), fname) // if it doesn't exist, copy it directly if (!path.exists()) { - Utils.copyFile(oFile, path); - return fname; + Utils.copyFile(oFile, path) + return fname } // if it's identical, reuse - if (Utils.fileChecksum(path).equals(csum)) { - return fname; + if (Utils.fileChecksum(path) == csum) { + return fname } // otherwise, increment the checksum in the filename - root = root + "-" + csum; + root = "$root-$csum" } } - - - /** - * String manipulation - * *********************************************************** - */ - - public List filesInStr(Long mid, String string) { - return filesInStr(mid, string, false); - } - - /** * Extract media filenames from an HTML string. * @@ -305,341 +203,316 @@ public class Media { * @param includeRemote If true will also include external http/https/ftp urls. * @return A list containing all the sound and image filenames found in the input string. */ - public List filesInStr(Long mid, String string, boolean includeRemote) { - List l = new ArrayList<>(); - Model model = mCol.getModels().get(mid); - List strings = new ArrayList<>(); - if (model.isCloze() && string.contains("{{c")) { + /** + * String manipulation + * *********************************************************** + */ + @JvmOverloads + fun filesInStr(mid: Long?, string: String, includeRemote: Boolean = false): List { + val l: MutableList = ArrayList() + val model = col.models.get(mid!!) + var strings: MutableList = ArrayList() + if (model!!.isCloze && string.contains("{{c")) { // if the field has clozes in it, we'll need to expand the // possibilities so we can render latex - strings = _expandClozes(string); + strings = _expandClozes(string) } else { - strings.add(string); + strings.add(string) } - - for (String s : strings) { + for (s in strings) { + @Suppress("NAME_SHADOWING") + var s = s // handle latex - s = LaTeX.mungeQA(s, mCol, model); + @KotlinCleanup("change to .map { }") + s = LaTeX.mungeQA(s!!, col, model) // extract filenames - Matcher m; - for (Pattern p : REGEXPS) { + var m: Matcher + for (p in REGEXPS) { // NOTE: python uses the named group 'fname'. Java doesn't have named groups, so we have to determine // the index based on which pattern we are using - int fnameIdx = p.equals(fSoundRegexps) ? 2 : p.equals(fImgAudioRegExpU) ? 2 : 3; - m = p.matcher(s); + val fnameIdx = if (p == fSoundRegexps) 2 else if (p == fImgAudioRegExpU) 2 else 3 + m = p.matcher(s) while (m.find()) { - String fname = m.group(fnameIdx); - boolean isLocal = !fRemotePattern.matcher(fname.toLowerCase(Locale.getDefault())).find(); + val fname = m.group(fnameIdx)!! + val isLocal = + !fRemotePattern.matcher(fname.lowercase(Locale.getDefault())).find() if (isLocal || includeRemote) { - l.add(fname); + l.add(fname) } } } } - return l; + return l } - - private List _expandClozes(String string) { - Set ords = new TreeSet<>(); - @SuppressWarnings("RegExpRedundantEscape") // In Android, } should be escaped - Matcher m = Pattern.compile("\\{\\{c(\\d+)::.+?\\}\\}").matcher(string); + private fun _expandClozes(string: String): MutableList { + val ords: MutableSet = TreeSet() + var m = // In Android, } should be escaped + Pattern.compile("\\{\\{c(\\d+)::.+?\\}\\}").matcher(string) while (m.find()) { - ords.add(m.group(1)); + ords.add(m.group(1)!!) } - ArrayList strings = new ArrayList<>(ords.size() + 1); - String clozeReg = TemplateFilters.CLOZE_REG; - - for (String ord : ords) { - StringBuffer buf = new StringBuffer(); - m = Pattern.compile(String.format(Locale.US, clozeReg, ord)).matcher(string); + val strings = ArrayList(ords.size + 1) + val clozeReg = TemplateFilters.CLOZE_REG + for (ord in ords) { + val buf = StringBuffer() + m = Pattern.compile(String.format(Locale.US, clozeReg, ord)).matcher(string) while (m.find()) { if (!TextUtils.isEmpty(m.group(4))) { - m.appendReplacement(buf, "[$4]"); + m.appendReplacement(buf, "[$4]") } else { - m.appendReplacement(buf, TemplateFilters.CLOZE_DELETION_REPLACEMENT); + m.appendReplacement(buf, TemplateFilters.CLOZE_DELETION_REPLACEMENT) } } - m.appendTail(buf); - String s = buf.toString().replaceAll(String.format(Locale.US, clozeReg, ".+?"), "$2"); - strings.add(s); + m.appendTail(buf) + val s = + buf.toString().replace(String.format(Locale.US, clozeReg, ".+?").toRegex(), "$2") + strings.add(s) } - strings.add(string.replaceAll(String.format(Locale.US, clozeReg, ".+?"), "$2")); - return strings; + strings.add(string.replace(String.format(Locale.US, clozeReg, ".+?").toRegex(), "$2")) + return strings } - /** * Strips a string from media references. * * @param txt The string to be cleared of media references. * @return The media-free string. */ - public String strip(String txt) { - for (Pattern p : REGEXPS) { - txt = p.matcher(txt).replaceAll(""); + @KotlinCleanup("return early and remove var") + fun strip(txt: String): String { + @Suppress("NAME_SHADOWING") + var txt = txt + for (p in REGEXPS) { + txt = p.matcher(txt).replaceAll("") } - return txt; + return txt } - - - public static String escapeImages(String string) { - return escapeImages(string, false); - } - - - /** - * Percent-escape UTF-8 characters in local image filenames. - * @param string The string to search for image references and escape the filenames. - * @return The string with the filenames of any local images percent-escaped as UTF-8. - */ - public static String escapeImages(String string, boolean unescape) { - for (Pattern p : Arrays.asList(fImgAudioRegExpQ, fImgAudioRegExpU)) { - Matcher m = p.matcher(string); - // NOTE: python uses the named group 'fname'. Java doesn't have named groups, so we have to determine - // the index based on which pattern we are using - int fnameIdx = p.equals(fImgAudioRegExpU) ? 2 : 3; - while (m.find()) { - String tag = m.group(0); - String fname = m.group(fnameIdx); - if (fRemotePattern.matcher(fname).find()) { - //don't do any escaping if remote image - } else { - if (unescape) { - string = string.replace(tag,tag.replace(fname, Uri.decode(fname))); - } else { - string = string.replace(tag,tag.replace(fname, Uri.encode(fname, "/"))); - } - } - } - } - return string; - } - - /* Rebuilding DB *********************************************************** */ - /** * Finds missing, unused and invalid media files * * @return A list containing three lists of files (missingFiles, unusedFiles, invalidFiles) */ - public @NonNull List> check() { - return check(null); + open fun check(): List> { + return check(null) } - - private @NonNull List> check(File[] local) { - File mdir = new File(dir()); + private fun check(local: Array?): List> { + val mdir = File(dir()) // gather all media references in NFC form - Set allRefs = new HashSet<>(); - try (Cursor cur = mCol.getDb().query("select id, mid, flds from notes")) { + val allRefs: MutableSet = HashSet() + col.db.query("select id, mid, flds from notes").use { cur -> while (cur.moveToNext()) { - long nid = cur.getLong(0); - long mid = cur.getLong(1); - String flds = cur.getString(2); - List noteRefs = filesInStr(mid, flds); + val nid = cur.getLong(0) + val mid = cur.getLong(1) + val flds = cur.getString(2) + var noteRefs = filesInStr(mid, flds) // check the refs are in NFC - for (String f : noteRefs) { + @KotlinCleanup("simplify with first {}") + for (f in noteRefs) { // if they're not, we'll need to fix them first - if (!f.equals(Utils.nfcNormalized(f))) { - _normalizeNoteRefs(nid); - noteRefs = filesInStr(mid, flds); - break; + if (f != Utils.nfcNormalized(f)) { + _normalizeNoteRefs(nid) + noteRefs = filesInStr(mid, flds) + break } } - allRefs.addAll(noteRefs); + allRefs.addAll(noteRefs) } } // loop through media directory - List unused = new ArrayList<>(); - List invalid = new ArrayList<>(); - File[] files; - if (local == null) { - files = mdir.listFiles(); - } else { - files = local; - } - boolean renamedFiles = false; - for (File file : files) { + val unused: MutableList = ArrayList() + val invalid: List = ArrayList() + val files: Array + files = local ?: mdir.listFiles()!! + var renamedFiles = false + for (file in files) { + @Suppress("NAME_SHADOWING") + var file = file if (local == null) { - if (file.isDirectory()) { + if (file.isDirectory) { // ignore directories - continue; + continue } } - if (file.getName().startsWith("_")) { + if (file.name.startsWith("_")) { // leading _ says to ignore file - continue; + continue } - File nfcFile = new File(dir(), Utils.nfcNormalized(file.getName())); + val nfcFile = File(dir(), Utils.nfcNormalized(file.name)) // we enforce NFC fs encoding if (local == null) { - if (!file.getName().equals(nfcFile.getName())) { + if (file.name != nfcFile.name) { // delete if we already have the NFC form, otherwise rename - if (nfcFile.exists()) { - file.delete(); - renamedFiles = true; + renamedFiles = if (nfcFile.exists()) { + file.delete() + true } else { - file.renameTo(nfcFile); - renamedFiles = true; + file.renameTo(nfcFile) + true } - file = nfcFile; + file = nfcFile } } // compare - if (!allRefs.contains(nfcFile.getName())) { - unused.add(file.getName()); + if (!allRefs.contains(nfcFile.name)) { + unused.add(file.name) } else { - allRefs.remove(nfcFile.getName()); + allRefs.remove(nfcFile.name) } } // if we renamed any files to nfc format, we must rerun the check // to make sure the renamed files are not marked as unused if (renamedFiles) { - return check(local); + return check(local) } - List noHave = new ArrayList<>(); - for (String x : allRefs) { + @KotlinCleanup(".filter { }") + val noHave: MutableList = ArrayList() + for (x in allRefs) { if (!x.startsWith("_")) { - noHave.add(x); + noHave.add(x) } } // make sure the media DB is valid try { - findChanges(); - } catch (SQLException ignored) { - Timber.w(ignored); - _deleteDB(); + findChanges() + } catch (ignored: SQLException) { + Timber.w(ignored) + _deleteDB() } - List> result = new ArrayList<>(3); - result.add(noHave); - result.add(unused); - result.add(invalid); - return result; + @KotlinCleanup("return listOf") + val result: MutableList> = ArrayList(3) + result.add(noHave) + result.add(unused) + result.add(invalid) + return result } - - private void _normalizeNoteRefs(long nid) { - Note note = mCol.getNote(nid); - String[] flds = note.getFields(); - for (int c = 0; c < flds.length; c++) { - String fld = flds[c]; - String nfc = Utils.nfcNormalized(fld); - if (!nfc.equals(fld)) { - note.setField(c, nfc); + private fun _normalizeNoteRefs(nid: Long) { + val note = col.getNote(nid) + val flds = note.fields + @KotlinCleanup("improve") + for (c in flds.indices) { + val fld = flds[c] + val nfc = Utils.nfcNormalized(fld) + if (nfc != fld) { + note.setField(c, nfc) } } - note.flush(); + note.flush() } - /** * Copying on import * *********************************************************** */ - - public boolean have(String fname) { - return new File(dir(), fname).exists(); + open fun have(fname: String): Boolean { + return File(dir(), fname).exists() } /** * Illegal characters and paths * *********************************************************** */ - - public String stripIllegal(String str) { - Matcher m = fIllegalCharReg.matcher(str); - return m.replaceAll(""); + @KotlinCleanup("one line function") + fun stripIllegal(str: String): String { + val m = fIllegalCharReg.matcher(str) + return m.replaceAll("") } - - public boolean hasIllegal(String str) { - Matcher m = fIllegalCharReg.matcher(str); - return m.find(); + @KotlinCleanup("one line function") + fun hasIllegal(str: String): Boolean { + val m = fIllegalCharReg.matcher(str) + return m.find() } - public String cleanFilename(String fname) { - fname = stripIllegal(fname); - fname = _cleanWin32Filename(fname); - fname = _cleanLongFilename(fname); - if ("".equals(fname)) { - fname = "renamed"; + @KotlinCleanup("fix reassignment") + fun cleanFilename(fname: String): String { + @Suppress("NAME_SHADOWING") + var fname = fname + fname = stripIllegal(fname) + fname = _cleanWin32Filename(fname) + fname = _cleanLongFilename(fname) + if ("" == fname) { + fname = "renamed" } - - return fname; + return fname } /** This method only change things on windows. So it's the - * identity here. */ - private String _cleanWin32Filename(String fname) { - return fname; + * identity here. */ + private fun _cleanWin32Filename(fname: String): String { + return fname } - private String _cleanLongFilename(String fname) { + @KotlinCleanup("Fix reassignment") + private fun _cleanLongFilename(fname: String): String { /* a fairly safe limit that should work on typical windows paths and on eCryptfs partitions, even with a duplicate suffix appended */ - int nameMax = 136; - int pathMax = 1024; // 240 for windows + @Suppress("NAME_SHADOWING") + var fname = fname + var nameMax = 136 + val pathMax = 1024 // 240 for windows // cap nameMax based on absolute path - int dirLen = fname.length();// ideally, name should be normalized. Without access to nio.Paths library, it's hard to do it really correctly. This is still a better approximation than nothing. - int remaining = pathMax - dirLen; - nameMax = min(remaining, nameMax); - Assert.that(nameMax>0, "The media directory is maximally long. There is no more length available for file name."); - - if (fname.length() > nameMax) { - int lastSlash = fname.indexOf("/"); - int lastDot = fname.indexOf("."); + val dirLen = + fname.length // ideally, name should be normalized. Without access to nio.Paths library, it's hard to do it really correctly. This is still a better approximation than nothing. + val remaining = pathMax - dirLen + nameMax = min(remaining, nameMax) + Assert.that( + nameMax > 0, + "The media directory is maximally long. There is no more length available for file name." + ) + if (fname.length > nameMax) { + val lastSlash = fname.indexOf("/") + val lastDot = fname.indexOf(".") if (lastDot == -1 || lastDot < lastSlash) { // no dot, or before last slash - fname = fname.substring(0, nameMax); + fname = fname.substring(0, nameMax) } else { - String ext = fname.substring(lastDot+1); - String head = fname.substring(0, lastDot); - int headMax = nameMax - ext.length(); - head = head.substring(0, headMax); - fname = head + ext; - Assert.that (fname.length() <= nameMax, "The length of the file is greater than the maximal name value."); + val ext = fname.substring(lastDot + 1) + var head = fname.substring(0, lastDot) + val headMax = nameMax - ext.length + head = head.substring(0, headMax) + fname = head + ext + Assert.that( + fname.length <= nameMax, + "The length of the file is greater than the maximal name value." + ) } } - - return fname; + return fname } - /* Tracking changes *********************************************************** */ - /** * Scan the media directory if it's changed, and note any changes. */ - public void findChanges() { - findChanges(false); + fun findChanges() { + findChanges(false) } - /** * @param force Unconditionally scan the media directory for changes (i.e., ignore differences in recorded and current - * directory mod times). Use this when rebuilding the media database. + * directory mod times). Use this when rebuilding the media database. */ - public void findChanges(boolean force) { + open fun findChanges(force: Boolean) { if (force || _changed() != null) { - _logChanges(); + _logChanges() } } - - public boolean haveDirty() { - return mDb.queryScalar("select 1 from media where dirty=1 limit 1") > 0; + fun haveDirty(): Boolean { + return db!!.queryScalar("select 1 from media where dirty=1 limit 1") > 0 } - /** * Returns the number of seconds from epoch since the last modification to the file in path. Important: this method * does not automatically append the root media directory to the path; the FULL path of the file must be specified. @@ -647,280 +520,276 @@ public class Media { * @param path The path to the file we are checking. path can be a file or a directory. * @return The number of seconds (rounded down). */ - private long _mtime(String path) { - File f = new File(path); - return f.lastModified() / 1000; + private fun _mtime(path: String): Long { + val f = File(path) + return f.lastModified() / 1000 } - - private String _checksum(String path) { - return Utils.fileChecksum(path); + private fun _checksum(path: String): String { + return Utils.fileChecksum(path) } - /** * Return dir mtime if it has changed since the last findChanges() * Doesn't track edits, but user can add or remove a file to update - * + * * @return The modification time of the media directory if it has changed since the last call of findChanges(). If - * it hasn't, it returns null. + * it hasn't, it returns null. */ - public Long _changed() { - long mod = mDb.queryLongScalar("select dirMod from meta"); - long mtime = _mtime(dir()); - if (mod != 0 && mod == mtime) { - return null; - } - return mtime; + fun _changed(): Long? { + val mod = db!!.queryLongScalar("select dirMod from meta") + val mtime = _mtime(dir()) + return if (mod != 0L && mod == mtime) { + null + } else mtime } - - private void _logChanges() { - Pair, List> result = _changes(); - List added = result.first; - List removed = result.second; - ArrayList media = new ArrayList<>(added.size() + removed.size()); - for (String f : added) { - String path = new File(dir(), f).getAbsolutePath(); - long mt = _mtime(path); - media.add(new Object[] { f, _checksum(path), mt, 1 }); + @KotlinCleanup("destructure directly val (added, removed) = _changes()") + private fun _logChanges() { + val result = _changes() + val added = result.first + val removed = result.second + val media = ArrayList>(added.size + removed.size) + for (f in added) { + val path = File(dir(), f).absolutePath + val mt = _mtime(path) + media.add(arrayOf(f, _checksum(path), mt, 1)) } - for (String f : removed) { - media.add(new Object[] { f, null, 0, 1}); + for (f in removed) { + media.add(arrayOf(f, null, 0, 1)) } // update media db - mDb.executeMany("insert or replace into media values (?,?,?,?)", media); - mDb.execute("update meta set dirMod = ?", _mtime(dir())); - mDb.commit(); + @KotlinCleanup("scope function on db") + db!!.executeMany("insert or replace into media values (?,?,?,?)", media) + db!!.execute("update meta set dirMod = ?", _mtime(dir())) + db!!.commit() } - - private Pair, List> _changes() { - Map cache = HashUtil.HashMapInit(mDb.queryScalar("SELECT count() FROM media WHERE csum IS NOT NULL")); - try (Cursor cur = mDb.query("select fname, csum, mtime from media where csum is not null")) { - while (cur.moveToNext()) { - String name = cur.getString(0); - String csum = cur.getString(1); - long mod = cur.getLong(2); - cache.put(name, new Object[] { csum, mod, false }); + private fun _changes(): Pair, List> { + val cache: MutableMap> = HashMapInit( + db!!.queryScalar("SELECT count() FROM media WHERE csum IS NOT NULL") + ) + try { + db!!.query("select fname, csum, mtime from media where csum is not null").use { cur -> + while (cur.moveToNext()) { + val name = cur.getString(0) + val csum = cur.getString(1) + val mod = cur.getLong(2) + cache[name] = arrayOf(csum, mod, false) + } } - } catch (SQLException e) { - throw new RuntimeException(e); + } catch (e: SQLException) { + throw RuntimeException(e) } - List added = new ArrayList<>(); - List removed = new ArrayList<>(); + val added: MutableList = ArrayList() + val removed: MutableList = ArrayList() // loop through on-disk files - for (File f : new File(dir()).listFiles()) { + for (f in File(dir()).listFiles()!!) { // ignore directories and thumbs.db - if (f.isDirectory()) { - continue; + if (f.isDirectory) { + continue } - String fname = f.getName(); - if ("thumbs.db".equalsIgnoreCase(fname)) { - continue; + val fname = f.name + if ("thumbs.db".equals(fname, ignoreCase = true)) { + continue } // and files with invalid chars if (hasIllegal(fname)) { - continue; + continue } // empty files are invalid; clean them up and continue - long sz = f.length(); - if (sz == 0) { - f.delete(); - continue; + val sz = f.length() + if (sz == 0L) { + f.delete() + continue } - if (sz > 100*1024*1024) { - mCol.log("ignoring file over 100MB", f); - continue; + if (sz > 100 * 1024 * 1024) { + col.log("ignoring file over 100MB", f) + continue } // check encoding - String normf = Utils.nfcNormalized(fname); - if (!fname.equals(normf)) { + val normf = Utils.nfcNormalized(fname) + if (fname != normf) { // wrong filename encoding which will cause sync errors - File nf = new File(dir(), normf); + val nf = File(dir(), normf) if (nf.exists()) { - f.delete(); + f.delete() } else { - f.renameTo(nf); + f.renameTo(nf) } } // newly added? if (!cache.containsKey(fname)) { - added.add(fname); + added.add(fname) } else { // modified since last time? - if (_mtime(f.getAbsolutePath()) != (Long) cache.get(fname)[1]) { + if (_mtime(f.absolutePath) != cache[fname]!![1] as Long) { // and has different checksum? - if (!_checksum(f.getAbsolutePath()).equals(cache.get(fname)[0])) { - added.add(fname); + if (_checksum(f.absolutePath) != cache[fname]!![0]) { + added.add(fname) } } // mark as used - cache.get(fname)[2] = true; + cache[fname]!![2] = true } } // look for any entries in the cache that no longer exist on disk - for (Map.Entry entry : cache.entrySet()) { - if (!((Boolean) entry.getValue()[2])) { - removed.add(entry.getKey()); + for ((key, value) in cache) { + if (!(value[2] as Boolean)) { + removed.add(key) } } - return new Pair<>(added, removed); + return Pair(added, removed) } - /** * Syncing related * *********************************************************** */ - - public int lastUsn() { - return mDb.queryScalar("select lastUsn from meta"); + fun lastUsn(): Int { + return db!!.queryScalar("select lastUsn from meta") } - - public void setLastUsn(int usn) { - mDb.execute("update meta set lastUsn = ?", usn); - mDb.commit(); + fun setLastUsn(usn: Int) { + db!!.execute("update meta set lastUsn = ?", usn) + db!!.commit() } - - public Pair syncInfo(String fname) { - try (Cursor cur = mDb.query("select csum, dirty from media where fname=?", fname)) { - if (cur.moveToNext()) { - String csum = cur.getString(0); - int dirty = cur.getInt(1); - return new Pair<>(csum, dirty); + fun syncInfo(fname: String?): Pair { + db!!.query("select csum, dirty from media where fname=?", fname!!).use { cur -> + return if (cur.moveToNext()) { + val csum = cur.getString(0) + val dirty = cur.getInt(1) + Pair(csum, dirty) } else { - return new Pair<>(null, 0); + Pair(null, 0) } } } - - public void markClean(List fnames) { - for (String fname : fnames) { - mDb.execute("update media set dirty=0 where fname=?", fname); + fun markClean(fnames: List) { + for (fname in fnames) { + db!!.execute("update media set dirty=0 where fname=?", fname!!) } } - - public void syncDelete(String fname) { - File f = new File(dir(), fname); + fun syncDelete(fname: String) { + val f = File(dir(), fname) if (f.exists()) { - f.delete(); + f.delete() } - mDb.execute("delete from media where fname=?", fname); + db!!.execute("delete from media where fname=?", fname) } - - public int mediacount() { - return mDb.queryScalar("select count() from media where csum is not null"); + fun mediacount(): Int { + return db!!.queryScalar("select count() from media where csum is not null") } - - public int dirtyCount() { - return mDb.queryScalar("select count() from media where dirty=1"); + fun dirtyCount(): Int { + return db!!.queryScalar("select count() from media where dirty=1") } - - public void forceResync() { - mDb.execute("delete from media"); - mDb.execute("update meta set lastUsn=0,dirMod=0"); - mDb.execute("vacuum"); - mDb.execute("analyze"); - mDb.commit(); + @KotlinCleanup("scope function on db") + open fun forceResync() { + db!!.execute("delete from media") + db!!.execute("update meta set lastUsn=0,dirMod=0") + db!!.execute("vacuum") + db!!.execute("analyze") + db!!.commit() } - - /* * Media syncing: zips * *********************************************************** */ - /** * Unlike python, our temp zip file will be on disk instead of in memory. This avoids storing * potentially large files in memory which is not feasible with Android's limited heap space. - *

+ * + * * Notes: - *

+ * + * * - The maximum size of the changes zip is decided by the constant SYNC_ZIP_SIZE. If a media file exceeds this * limit, only that file (in full) will be zipped to be sent to the server. - *

+ * + * * - This method will be repeatedly called from MediaSyncer until there are no more files (marked "dirty" in the DB) * to send. - *

+ * + * * - Since AnkiDroid avoids scanning the media directory on every sync, it is possible for a file to be marked as a * new addition but actually have been deleted (e.g., with a file manager). In this case we skip over the file * and mark it as removed in the database. (This behaviour differs from the desktop client). - *

+ * + * */ - public Pair> mediaChangesZip() { - File f = new File(mCol.getPath().replaceFirst("collection\\.anki2$", "tmpSyncToServer.zip")); - List fnames = new ArrayList<>(); - try (ZipOutputStream z = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(f))); - Cursor cur = mDb.query( - "select fname, csum from media where dirty=1 limit " + Consts.SYNC_MAX_FILES) - ) { - z.setMethod(ZipOutputStream.DEFLATED); + fun mediaChangesZip(): Pair> { + val f = File(col.path.replaceFirst("collection\\.anki2$".toRegex(), "tmpSyncToServer.zip")) + val fnames: MutableList = ArrayList() + try { + ZipOutputStream(BufferedOutputStream(FileOutputStream(f))).use { z -> + db!!.query( + "select fname, csum from media where dirty=1 limit " + Consts.SYNC_MAX_FILES + ).use { cur -> + z.setMethod(ZipOutputStream.DEFLATED) - // meta is a list of (fname, zipName), where zipName of null is a deleted file - // NOTE: In python, meta is a list of tuples that then gets serialized into json and added - // to the zip as a string. In our version, we use JSON objects from the start to avoid the - // serialization step. Instead of a list of tuples, we use JSONArrays of JSONArrays. - JSONArray meta = new JSONArray(); - int sz = 0; - byte[] buffer = new byte[2048]; - - - for (int c = 0; cur.moveToNext(); c++) { - String fname = cur.getString(0); - String csum = cur.getString(1); - fnames.add(fname); - String normName = Utils.nfcNormalized(fname); - - if (!TextUtils.isEmpty(csum)) { - try { - mCol.log("+media zip " + fname); - File file = new File(dir(), fname); - BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file), 2048); - z.putNextEntry(new ZipEntry(Integer.toString(c))); - int count = 0; - while ((count = bis.read(buffer, 0, 2048)) != -1) { - z.write(buffer, 0, count); + // meta is a list of (fname, zipName), where zipName of null is a deleted file + // NOTE: In python, meta is a list of tuples that then gets serialized into json and added + // to the zip as a string. In our version, we use JSON objects from the start to avoid the + // serialization step. Instead of a list of tuples, we use JSONArrays of JSONArrays. + val meta = JSONArray() + var sz = 0 + val buffer = ByteArray(2048) + var c = 0 + while (cur.moveToNext()) { + val fname = cur.getString(0) + val csum = cur.getString(1) + fnames.add(fname) + val normName = Utils.nfcNormalized(fname) + if (!TextUtils.isEmpty(csum)) { + try { + col.log("+media zip $fname") + val file = File(dir(), fname) + val bis = BufferedInputStream(FileInputStream(file), 2048) + z.putNextEntry(ZipEntry(Integer.toString(c))) + @KotlinCleanup("improve") + var count: Int + while (bis.read(buffer, 0, 2048).also { count = it } != -1) { + z.write(buffer, 0, count) + } + z.closeEntry() + bis.close() + meta.put(JSONArray().put(normName).put(Integer.toString(c))) + sz += file.length().toInt() + } catch (e: FileNotFoundException) { + Timber.w(e) + // A file has been marked as added but no longer exists in the media directory. + // Skip over it and mark it as removed in the db. + removeFile(fname) + } + } else { + col.log("-media zip $fname") + meta.put(JSONArray().put(normName).put("")) } - z.closeEntry(); - bis.close(); - meta.put(new JSONArray().put(normName).put(Integer.toString(c))); - sz += file.length(); - } catch (FileNotFoundException e) { - Timber.w(e); - // A file has been marked as added but no longer exists in the media directory. - // Skip over it and mark it as removed in the db. - removeFile(fname); + if (sz >= Consts.SYNC_MAX_BYTES) { + break + } + c++ } - } else { - mCol.log("-media zip " + fname); - meta.put(new JSONArray().put(normName).put("")); - } - if (sz >= Consts.SYNC_MAX_BYTES) { - break; + z.putNextEntry(ZipEntry("_meta")) + z.write(Utils.jsonToString(meta).toByteArray()) + z.closeEntry() + // Don't leave lingering temp files if the VM terminates. + f.deleteOnExit() + return Pair(f, fnames) } } - - z.putNextEntry(new ZipEntry("_meta")); - z.write(Utils.jsonToString(meta).getBytes()); - z.closeEntry(); - // Don't leave lingering temp files if the VM terminates. - f.deleteOnExit(); - return new Pair<>(f, fnames); - } catch (IOException e) { - Timber.e(e, "Failed to create media changes zip: "); - throw new RuntimeException(e); + } catch (e: IOException) { + Timber.e(e, "Failed to create media changes zip: ") + throw RuntimeException(e) } } - /** * Extract zip data; return the number of files extracted. Unlike the python version, this method consumes a * ZipFile stored on disk instead of a String buffer. Holding the entire downloaded data in memory is not feasible @@ -928,129 +797,210 @@ public class Media { * * This method closes the file before it returns. */ - public int addFilesFromZip(ZipFile z) throws IOException { - try { + @Throws(IOException::class) + fun addFilesFromZip(z: ZipFile): Int { + return try { // get meta info first - JSONObject meta = new JSONObject(Utils.convertStreamToString(z.getInputStream(z.getEntry("_meta")))); + val meta = + JSONObject(Utils.convertStreamToString(z.getInputStream(z.getEntry("_meta")))) // then loop through all files - int cnt = 0; - ArrayList zipEntries = Collections.list(z.entries()); - List media = new ArrayList<>(zipEntries.size()); - for (ZipEntry i : zipEntries) { - String fileName = i.getName(); - if ("_meta".equals(fileName)) { - // ignore previously-retrieved meta - continue; + var cnt = 0 + val zipEntries = Collections.list(z.entries()) + val media: MutableList> = ArrayList(zipEntries.size) + for (i in zipEntries) { + val fileName = i.name + if ("_meta" == fileName) { + // ignore previously-retrieved meta + continue } - String name = meta.getString(fileName); + var name = meta.getString(fileName) // normalize name for platform - name = Utils.nfcNormalized(name); + name = Utils.nfcNormalized(name) // save file - String destPath = (dir() + File.separator) + name; - try (InputStream zipInputStream = z.getInputStream(i)) { - Utils.writeToFile(zipInputStream, destPath); - } - String csum = Utils.fileChecksum(destPath); + val destPath = dir() + File.separator + name + z.getInputStream(i) + .use { zipInputStream -> Utils.writeToFile(zipInputStream, destPath) } + val csum = Utils.fileChecksum(destPath) // update db - media.add(new Object[] {name, csum, _mtime(destPath), 0}); - cnt += 1; + media.add(arrayOf(name, csum, _mtime(destPath), 0)) + cnt += 1 } if (!media.isEmpty()) { - mDb.executeMany("insert or replace into media values (?,?,?,?)", media); + db!!.executeMany("insert or replace into media values (?,?,?,?)", media) } - return cnt; + cnt } finally { - z.close(); + z.close() } } - - /* * *********************************************************** * The methods below are not in LibAnki. * *********************************************************** */ - - /** - * Used by unit tests only. - */ - public DB getDb() { - return mDb; - } - - - /** - * Used by other classes to determine the index of a regular expression group named "fname" - * (Anki2Importer needs this). This is needed because we didn't implement the "transformNames" - * function and have delegated its job to the caller of this class. - */ - public static int indexOfFname(Pattern p) { - return p.equals(fSoundRegexps) ? 2 : p.equals(fImgAudioRegExpU) ? 2 : 3; - } - - /** * Add an entry into the media database for file named fname, or update it * if it already exists. */ - public void markFileAdd(String fname) { - Timber.d("Marking media file addition in media db: %s", fname); - String path = new File(dir(), fname).getAbsolutePath(); - mDb.execute("insert or replace into media values (?,?,?,?)", - fname, _checksum(path), _mtime(path), 1); + open fun markFileAdd(fname: String) { + Timber.d("Marking media file addition in media db: %s", fname) + val path = File(dir(), fname).absolutePath + db!!.execute( + "insert or replace into media values (?,?,?,?)", + fname, _checksum(path), _mtime(path), 1 + ) } - /** * Remove a file from the media directory if it exists and mark it as removed in the media database. */ - public void removeFile(String fname) { - File f = new File(dir(), fname); + open fun removeFile(fname: String) { + val f = File(dir(), fname) if (f.exists()) { - f.delete(); + f.delete() } - Timber.d("Marking media file removal in media db: %s", fname); - mDb.execute("insert or replace into media values (?,?,?,?)", - fname, null, 0, 1); + Timber.d("Marking media file removal in media db: %s", fname) + db!!.execute( + "insert or replace into media values (?,?,?,?)", + fname, null, 0, 1 + ) } - /** * @return True if the media db has not been populated yet. */ - public boolean needScan() { - long mod = mDb.queryLongScalar("select dirMod from meta"); - return mod == 0; + fun needScan(): Boolean { + val mod = db!!.queryLongScalar("select dirMod from meta") + return mod == 0L } - - public void rebuildIfInvalid() throws IOException { + @Throws(IOException::class) + open fun rebuildIfInvalid() { try { - _changed(); - return; - } catch (Exception e) { + _changed() + return + } catch (e: Exception) { if (!ExceptionUtil.containsMessage(e, "no such table: meta")) { - throw e; + throw e } - CrashReportService.sendExceptionReport(e, "media::rebuildIfInvalid"); + CrashReportService.sendExceptionReport(e, "media::rebuildIfInvalid") // TODO: We don't know the root cause of the missing meta table - Timber.w(e, "Error accessing media database. Rebuilding"); + Timber.w(e, "Error accessing media database. Rebuilding") // continue below } - // Delete and recreate the file - mDb.getDatabase().close(); - - String path = mDb.getPath(); - Timber.i("Deleted %s", path); - - new File(path).delete(); - - mDb = DB.withAndroidFramework(mCol.getContext(), path); - _initDB(); + db!!.database.close() + val path = db!!.path + Timber.i("Deleted %s", path) + File(path).delete() + db = DB.withAndroidFramework(col.context, path) + _initDB() } + companion object { + // Upstream illegal chars defined on disallowed_char() + // in https://github.com/ankitects/anki/blob/main/rslib/src/media/files.rs + private val fIllegalCharReg = Pattern.compile("[\\[\\]><:\"/?*^\\\\|\\x00\\r\\n]") + private val fRemotePattern = Pattern.compile("(https?|ftp)://") + /* + * A note about the regular expressions below: the python code uses named groups for the image and sound patterns. + * Our version of Java doesn't support named groups, so we must use indexes instead. In the expressions below, the + * group names (e.g., ?P) have been stripped and a comment placed above indicating the index of the group + * name in the original. Refer to these indexes whenever the python code makes use of a named group. + */ + /** + * Group 1 = Contents of [sound:] tag + * Group 2 = "fname" + */ + // Regexes defined on https://github.com/ankitects/anki/blob/b403f20cae8fcdd7c3ff4c8d21766998e8efaba0/pylib/anki/media.py#L34-L45 + private val fSoundRegexps = Pattern.compile("(?i)(\\[sound:([^]]+)])") + // src element quoted case + /** + * Group 1 = Contents of `|