diff --git a/tools/cldr-apps/js/src/esm/cldrAccount.mjs b/tools/cldr-apps/js/src/esm/cldrAccount.mjs index f1a5df7968c..36056ac4274 100644 --- a/tools/cldr-apps/js/src/esm/cldrAccount.mjs +++ b/tools/cldr-apps/js/src/esm/cldrAccount.mjs @@ -763,14 +763,24 @@ function prettyLocaleList(locales) { } function getUserSeen(u) { - const when = u.data.active ? u.data.active : u.data.seen; - if (!when) { - return ""; - } + const when = (u.data.active ? u.data.active : u.data.seen) || "never"; const what = u.data.active ? "active" : "seen"; let html = "" + what + ": " + when + ""; if (what === "seen") { - html += "
" + u.data.lastlogin + ""; + let created = u.data.firstdate; + if (!created) { + created = "unknown date; never voted?"; + } else if (created < "2025-10-17") { + // Change 2023-05-04T20:00:00.000Z to 2023-05-04, for example. + created = "approximately " + created.replace(/T.+/, ""); + } + const lastlogin = u.data.lastlogin || "never"; + html += + "
" + + lastlogin + + "
(account created " + + created + + ")"; } return html; } diff --git a/tools/cldr-apps/js/src/esm/cldrAddUser.mjs b/tools/cldr-apps/js/src/esm/cldrAddUser.mjs index aecfd485e4c..a5af025dffb 100644 --- a/tools/cldr-apps/js/src/esm/cldrAddUser.mjs +++ b/tools/cldr-apps/js/src/esm/cldrAddUser.mjs @@ -73,11 +73,11 @@ async function getOrgLocales(orgName) { .doFetch(resource) .then(cldrAjax.handleFetchErrors) .then((r) => r.json()) - .then(setOrgLocales) + .then((json) => setOrgLocales(orgName, json)) .catch((e) => addError(`Error: ${e} getting org locales`)); } -function setOrgLocales(json) { +function setOrgLocales(orgName, json) { if (json.err) { cldrRetry.handleDisconnect(json.err, json, "", "Loading org locales"); return; @@ -87,6 +87,7 @@ function setOrgLocales(json) { ? Object.keys(cldrLoad.getTheLocaleMap().locmap.locales).join(" ") : json.locales; callbackToSetData({ + orgName, orgLocales, }); } diff --git a/tools/cldr-apps/js/src/views/AddUser.vue b/tools/cldr-apps/js/src/views/AddUser.vue index 7f116f9b213..925871b33d7 100644 --- a/tools/cldr-apps/js/src/views/AddUser.vue +++ b/tools/cldr-apps/js/src/views/AddUser.vue @@ -262,8 +262,8 @@ function setData(data) { if (data.orgObject) { setOrgData(data.orgObject); } - if (data.orgLocales) { - setOrgLocales(data.orgLocales); + if (data.orgName && data.orgLocales) { + setOrgLocales(data.orgName, data.orgLocales); } if (data.error) { justGotError.value = true; // See comment in errorsExist() @@ -303,7 +303,8 @@ function setOrgData(orgObject) { newUserOrg.value = ""; } -function setOrgLocales(orgLocales) { +function setOrgLocales(orgName, orgLocales) { + newUserOrg.value = orgName; if (!orgLocales) { console.error("No locales for organization " + newUserOrg.value); } diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DBUtils.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DBUtils.java index f44c0efe2d5..11aa7a9a5dc 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DBUtils.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DBUtils.java @@ -329,6 +329,18 @@ public static boolean tableHasColumn(Connection conn, String table, String colum > 0; } + public static boolean addColumnIfMissing( + Connection conn, String tableName, String columnName, String type) throws SQLException { + if (DBUtils.tableHasColumn(conn, tableName, columnName)) { + return false; // Did not create since it already exists + } + Statement s = conn.createStatement(); + String sql = "ALTER TABLE " + tableName + " ADD COLUMN " + columnName + " " + type; + s.execute(sql); + s.close(); + return true; + } + private static byte[] encode_u8(String what) { byte[] u8; if (what == null) { diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java index 3e1af037dc7..550a1396b1d 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java @@ -1511,18 +1511,8 @@ public synchronized void setupDB() { s.close(); s = null; // don't close twice. System.err.println("Created table " + DBUtils.Table.VOTE_VALUE); - } else if (!DBUtils.tableHasColumn( - conn, DBUtils.Table.VOTE_VALUE.toString(), VOTE_TYPE)) { - s = conn.createStatement(); - sql = - "ALTER TABLE " - + DBUtils.Table.VOTE_VALUE - + " ADD COLUMN " - + VOTE_TYPE - + " TINYINT NOT NULL"; - s.execute(sql); - s.close(); - s = null; + } else if (DBUtils.addColumnIfMissing( + conn, DBUtils.Table.VOTE_VALUE.toString(), VOTE_TYPE, "TINYINT NOT NULL")) { System.err.println( "Added column " + VOTE_TYPE + " to table " + DBUtils.Table.VOTE_VALUE); } diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyJSONWrapper.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyJSONWrapper.java index 24618650164..ed57fe59387 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyJSONWrapper.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyJSONWrapper.java @@ -91,7 +91,8 @@ public static JSONObject wrap(UserRegistry.User u) throws JSONException { .put("emailHash", u.getEmailHash()) .put("userlevelName", u.getLevel()) .put("org", u.org) - .put("time", u.last_connect); + .put("firstdate", u.firstdate) + .put("time", u.lastlogin); } public static JSONObject wrap(CheckCLDR check) throws JSONException { diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserList.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserList.java index c8e33dabdfd..c33217d5d6a 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserList.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserList.java @@ -446,8 +446,7 @@ private void putShownUser(JSONArray shownUsers, UserSettings u) throws JSONExcep (u.session == null) ? "" : SurveyMain.timeDiff(u.session.getLastBrowserCallMillisSinceEpoch()); - String seen = - (user.last_connect == null) ? "" : SurveyMain.timeDiff(user.last_connect.getTime()); + String seen = (user.lastlogin == null) ? "" : SurveyMain.timeDiff(user.lastlogin.getTime()); boolean havePermToChange = me.isAdminFor(user); boolean userCanDeleteUser = UserRegistry.userCanDeleteUser(me, user.id, user.userlevel); VoteResolver.Level level = VoteResolver.Level.fromSTLevel(user.userlevel); @@ -461,7 +460,8 @@ private void putShownUser(JSONArray shownUsers, UserSettings u) throws JSONExcep .put("havePermToChange", havePermToChange) .put("id", user.id) .put("intlocs", user.intlocs) - .put("lastlogin", user.last_connect) + .put("firstdate", user.firstdate) + .put("lastlogin", user.lastlogin) .put("locales", normalizeLocales(user.locales)) .put("badLocales", user.badLocales) .put("name", user.name) diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserRegistry.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserRegistry.java index c36f4390c8f..19b2d0ac773 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserRegistry.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserRegistry.java @@ -6,12 +6,19 @@ package org.unicode.cldr.web; +import static java.lang.Math.abs; + import com.google.gson.JsonSyntaxException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -19,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; import org.apache.commons.codec.digest.DigestUtils; @@ -152,27 +160,28 @@ public static String levelAsStr(int level) { public static final String CLDR_INTEREST = "cldr_interest"; - public static final String SQL_insertStmt = + private static final String SQL_insertStmt = "INSERT INTO " + CLDR_USERS - + "(userlevel,name,org,email,password,locales,lastlogin) " - + "VALUES(?,?,?,?,?,?,NULL)"; - public static final String SQL_queryStmt_FRO = - "SELECT id,name,userlevel,org,locales,intlocs,lastlogin from " + + "(userlevel,name,org,email,password,locales,firstdate,lastlogin) " + + "VALUES(?,?,?,?,?,?,?,NULL)"; + private static final String SQL_queryStmt_FRO = + "SELECT id,name,userlevel,org,locales,intlocs,firstdate,lastlogin FROM " + CLDR_USERS - + " where email=? AND password=?"; - public static final String SQL_queryIdStmt_FRO = - "SELECT name,org,email,userlevel,intlocs,locales,lastlogin,password from " + + " WHERE email=? AND password=?"; + private static final String SQL_queryIdStmt_FRO = + "SELECT name,org,email,userlevel,intlocs,locales,firstdate,lastlogin,password FROM " + CLDR_USERS - + " where id=?"; - public static final String SQL_queryEmailStmt_FRO = - "SELECT id,name,userlevel,org,locales,intlocs,lastlogin,password from " + + " WHERE id=?"; + private static final String SQL_queryEmailStmt_FRO = + "SELECT id,name,userlevel,org,locales,intlocs,firstdate,lastlogin,password FROM " + CLDR_USERS - + " where email=?"; - public static final String SQL_touchStmt = - "UPDATE " + CLDR_USERS + " set lastlogin=CURRENT_TIMESTAMP where id=?"; - public static final String SQL_removeIntLoc = "DELETE FROM " + CLDR_INTEREST + " WHERE uid=?"; - public static final String SQL_updateIntLoc = + + " WHERE email=?"; + + private static final String SQL_touchStmt = + "UPDATE " + CLDR_USERS + " SET lastlogin=CURRENT_TIMESTAMP WHERE id=?"; + private static final String SQL_removeIntLoc = "DELETE FROM " + CLDR_INTEREST + " WHERE uid=?"; + private static final String SQL_updateIntLoc = "INSERT INTO " + CLDR_INTEREST + " (uid,forum) VALUES(?,?)"; private UserSettingsData userSettings = null; @@ -185,14 +194,14 @@ public class User implements Comparable, UserInfo, JSONString { @Schema(description = "User ID") public int id; // id number - @Schema(description = "numeric userlevel") + @Schema(description = "Numeric userlevel") public int userlevel = LOCKED; // user level @Schema(hidden = true) private String password; // password @Schema(description = "User email") - public String email; // + public String email; @Schema(description = "User org") public String org; // organization @@ -200,13 +209,19 @@ public class User implements Comparable, UserInfo, JSONString { @Schema(description = "User name") public String name; // full name + @Schema( + description = "User account creation date, approximate if before 2025-10-17", + implementation = java.util.Date.class) + public Timestamp firstdate; + @Schema(name = "time", implementation = java.util.Date.class) - public java.sql.Timestamp last_connect; + // Here and in some http responses, lastlogin is named "time". + public java.sql.Timestamp lastlogin; @Schema(hidden = true) public String locales; - @Schema(name = "badLocales", description = "set of incorrectly specified locales") + @Schema(name = "badLocales", description = "Set of incorrectly specified locales") public String[] badLocales = null; @Schema(hidden = true) @@ -638,13 +653,13 @@ public void setLocales(String list, String rawList) { } else { locales = list; } - Set localeSet = new HashSet(); + Set localeSet = new HashSet<>(); for (final String s : LocaleNormalizer.splitToArray(list)) { if (!LocaleNormalizer.isAllLocales(s)) { localeSet.add(s); } } - Set rawSet = new HashSet(); + Set rawSet = new HashSet<>(); for (final String s : LocaleNormalizer.splitToArray(rawList)) { if (!LocaleNormalizer.isAllLocales(s)) { rawSet.add(s); @@ -652,7 +667,7 @@ public void setLocales(String list, String rawList) { } // take out the rawSet as un-normalized rawSet.removeAll(localeSet); - Set badSet = new TreeSet(); + Set badSet = new TreeSet<>(); badSet.addAll( rawSet); // anything still in the rawSet is bad somehow, it's un-normalized. // anything in the localeSet that's not an extant locale @@ -745,6 +760,10 @@ private void setupDB() throws SQLException { createUserTable(conn); conn.commit(); } else { + if (!DBUtils.tableHasColumn(conn, CLDR_USERS, "firstdate")) { + logger.warning("firstdate column was missing; calling addFirstDateColumn"); + addFirstDateColumn(conn); + } /* update table to DATETIME instead of TIMESTAMP */ Statement s = conn.createStatement(); sql = "alter table cldr_users change lastlogin lastlogin DATETIME"; @@ -812,41 +831,32 @@ private void createUserTable(Connection conn) throws SQLException { Statement s = conn.createStatement(); sql = - ("create table " + ("CREATE TABLE " + CLDR_USERS + "(id INT NOT NULL " + DBUtils.DB_SQL_IDENTITY + ", " - + "userlevel int not null, " + + "userlevel INT NOT NULL, " + "name " + DBUtils.DB_SQL_UNICODE - + " not null, " - + "email varchar(128) not null UNIQUE, " - + "org varchar(256) not null, " - + "password varchar(100) not null, " - + "audit varchar(1024) , " - + "locales varchar(1024) , " - + - // "prefs varchar(1024) , " + /* deprecated Dec 2010. Not used - // anywhere */ - "intlocs varchar(1024) , " - + // added apr 2006: ALTER table - // CLDR_USERS ADD COLUMN intlocs - // VARCHAR(1024) - "lastlogin " + + " NOT NULL, " + + "email VARCHAR(128) NOT NULL UNIQUE, " + + "org VARCHAR(256) NOT NULL, " + + "password VARCHAR(100) NOT NULL, " + + "audit VARCHAR(1024) , " + + "locales VARCHAR(1024) , " + + "intlocs VARCHAR(1024) , " + + "firstdate " + DBUtils.DB_SQL_TIMESTAMP0 - + // added may 2006: - // alter table - // CLDR_USERS ADD - // COLUMN lastlogin - // TIMESTAMP - (!DBUtils.db_Mysql ? ",primary key(id)" : "") + + " , lastlogin " + + DBUtils.DB_SQL_TIMESTAMP0 + + (!DBUtils.db_Mysql ? ",PRIMARY KEY(id)" : "") + ")"); s.execute(sql); sql = ("INSERT INTO " + CLDR_USERS - + "(userlevel,name,org,email,password) " + + "(userlevel,name,org,email,password,firstdate) " + "VALUES(" + ADMIN + "," @@ -857,6 +867,8 @@ private void createUserTable(Connection conn) throws SQLException { + "'," + "'" + SurveyMain.vap + + "', '" + + DBUtils.sqlNow() + "')"); s.execute(sql); SurveyLog.debug("DB: added user Admin"); @@ -941,8 +953,9 @@ public UserRegistry.User getInfo(int id) { u.intlocs = rs.getString(5); final String locales = rs.getString(6); u.setLocales(LocaleNormalizer.normalizeQuietly(locales), locales); - u.last_connect = rs.getTimestamp(7); - u.password = rs.getString(8); + u.firstdate = rs.getTimestamp(7); + u.lastlogin = rs.getTimestamp(8); + u.password = rs.getString(9); ret = u; // let it finish.. if (id >= arraySize) { @@ -1031,10 +1044,10 @@ public void touch(int id) { */ public UserRegistry.User get(String pass, String email, String ip, boolean letmein) throws LogoutException { - if ((email == null) || (email.length() == 0)) { + if ((email == null) || (email.isEmpty())) { return null; // nothing to do } - if (((pass != null && pass.length() == 0)) && !letmein) { + if (((pass != null && pass.isEmpty())) && !letmein) { return null; // nothing to do } @@ -1072,7 +1085,8 @@ public UserRegistry.User get(String pass, String email, String ip, boolean letme u.org = rs.getString(4); u.setLocales(rs.getString(5)); u.intlocs = rs.getString(6); - u.last_connect = rs.getTimestamp(7); + u.firstdate = rs.getTimestamp(7); + u.lastlogin = rs.getTimestamp(8); u.claSigned = (u.getCla() != null); // good so far.. @@ -1171,14 +1185,14 @@ public java.sql.PreparedStatement list(String organization, Connection conn) return DBUtils.prepareStatementForwardReadOnly( conn, "listAllUsers", - "SELECT id,userlevel,name,email,org,locales,intlocs,lastlogin FROM " + "SELECT id,userlevel,name,email,org,locales,intlocs,firstdate,lastlogin FROM " + CLDR_USERS + " ORDER BY org,userlevel,name "); } else { PreparedStatement ps = DBUtils.prepareStatementWithArgsFRO( conn, - "SELECT id,userlevel,name,email,org,locales,intlocs,lastlogin FROM " + "SELECT id,userlevel,name,email,org,locales,intlocs,firstdate,lastlogin FROM " + CLDR_USERS + " WHERE org=? ORDER BY org,userlevel,name"); ps.setString(1, organization); @@ -1774,6 +1788,7 @@ public User newUser(WebContext ctx, User u) { u.name = u.name.replace('\'', '_'); final String locales = (u.locales == null) ? "" : u.locales.replace('\'', '_'); u.setLocales(LocaleNormalizer.normalizeQuietly(locales), locales); + u.firstdate = DBUtils.sqlNow(); Connection conn = null; PreparedStatement insertStmt = null; @@ -1786,6 +1801,7 @@ public User newUser(WebContext ctx, User u) { insertStmt.setString(4, u.email); insertStmt.setString(5, u.getPassword()); insertStmt.setString(6, u.locales); + insertStmt.setTimestamp(7, u.firstdate); if (!insertStmt.execute()) { if (!hushUserMessages) logger.info("Added."); conn.commit(); @@ -2121,7 +2137,7 @@ public synchronized Map getVoterToInfo() { conn = DBUtils.getInstance().getAConnection(); ps = list(null, conn); rs = ps.executeQuery(); - // id,userlevel,name,email,org,locales,intlocs,lastlogin + // id,userlevel,name,email,org,locales,intlocs,firstdate,lastlogin while (rs.next()) { // We don't go through the cache, because not all users may // be loaded. @@ -2134,7 +2150,8 @@ public synchronized Map getVoterToInfo() { u.org = rs.getString(5); u.setLocales(rs.getString(6)); u.intlocs = rs.getString(7); - u.last_connect = rs.getTimestamp(8); + u.firstdate = rs.getTimestamp(8); + u.lastlogin = rs.getTimestamp(9); // now, map it to a UserInfo VoterInfo v = u.createVoterInfo(); @@ -2240,7 +2257,7 @@ private Set getAnonymousUsersFromDb() { conn = DBUtils.getInstance().getAConnection(); ps = list(null, conn); rs = ps.executeQuery(); - // id,userlevel,name,email,org,locales,intlocs,lastlogin + // id,userlevel,name,email,org,locales,intlocs,firstdate,lastlogin while (rs.next()) { int userlevel = rs.getInt(2); if (userlevel == ANONYMOUS) { @@ -2251,7 +2268,8 @@ private Set getAnonymousUsersFromDb() { u.org = rs.getString(5); u.setLocales(rs.getString(6)); u.intlocs = rs.getString(7); - u.last_connect = rs.getTimestamp(8); + u.firstdate = rs.getTimestamp(8); + u.lastlogin = rs.getTimestamp(9); u.claSigned = true; set.add(u); } @@ -2424,4 +2442,399 @@ private void normalizeUserTableOrgs() { DBUtils.close(rs, ps, conn); } } + + /** + * Create and populate the firstdate column. For each user, set firstdate based on the oldest + * version in which they voted, according to vote tables from version 25 and later; users who + * never voted get values based on user ID numbers, which are in chronological order. This is + * expected to be a one-time operation performed 2015-10 by ST at startup before the start of + * version v49. Subsequently new users will get firstdate set when their accounts are created. + * + * @param conn the db connection + * @throws SQLException if error + */ + private void addFirstDateColumn(Connection conn) throws SQLException { + logger.warning("Starting addFirstDateColumn"); + final long timeStart = System.currentTimeMillis(); + // First get the list of all users from the db users table, and map each one to + // placeholder version zero. + final TreeMap userFirstVersion = new TreeMap<>(); + final String sql = "SELECT id FROM " + CLDR_USERS; + ResultSet rs = null; + PreparedStatement ps = null; + try { + ps = conn.prepareStatement(sql); + rs = ps.executeQuery(); + while (rs.next()) { + int userId = rs.getInt(1); + if (userId < 1 || userId > 3300) { + logger.warning("UNEXPECTED userId: " + userId); + } + userFirstVersion.put(userId, 0); + } + } finally { + DBUtils.close(rs, ps); + } + // Next get the first version for each user in the map. + final Map verToDate = mapVersionToFirstDate(conn); + Map verToLastUser = new TreeMap<>(); + getFirstVersions(conn, userFirstVersion, verToDate, verToLastUser); + improveFirstVersions(userFirstVersion, verToLastUser); + + // Finally, create the new firstdate column, and populate it using userFirstDate. + if (DBUtils.addColumnIfMissing(conn, CLDR_USERS, "firstdate", DBUtils.DB_SQL_TIMESTAMP0)) { + logger.warning("firstdate column was missing; added; calling populateFirstDateColumn"); + populateFirstDateColumn(conn, userFirstVersion, verToDate); + } else { + logger.severe("addFirstDateColumn failed to add column"); + } + long elapsedTime = (System.currentTimeMillis() - timeStart) / (1000 * 60); + logger.warning("Finished addFirstDateColumn, elapsed time = " + elapsedTime + " minutes"); + } + + /** + * Replace the placeholder time stamps in the given map with the first date for each user + * + * @param conn the db connection + * @param userFirstVersion map from user ID to version number, to be modified + * @param verToDate map from version number to date + * @param verToLastUser map from version number to the highest user ID that voted in that + * version + * @throws SQLException if error + */ + private void getFirstVersions( + Connection conn, + Map userFirstVersion, + Map verToDate, + Map verToLastUser) + throws SQLException { + long timeStart = System.currentTimeMillis(); + int fixedCountAllVersions = 0; + for (int ver : verToDate.keySet()) { + int fixedCountThisVersion = 0; + String voteTable = + DBUtils.Table.VOTE_VALUE.forVersion(Integer.toString(ver), false).toString(); + if (DBUtils.hasTable(voteTable)) { + // Basically "SELECT DISTINCT submitter FROM " + voteTable, but + // skip votes copied from one user to another ("WHERE NOT EXISTS ..."). + // Ignore the locale (omit "b.locale = a.locale AND") since copying has + // evidently occurred between locales such as "nb" and "no"; assume that + // exactly matching timestamps (last_mod) for the same xpath are not coincidental. + final String sql = + "SELECT DISTINCT submitter FROM " + + voteTable + + " AS a WHERE NOT EXISTS (SELECT * FROM " + + voteTable + + " AS b WHERE b.xpath = a.xpath AND b.last_mod = a.last_mod " + + "AND b.submitter < a.submitter)"; + logger.warning("sql = " + sql); + PreparedStatement ps = null; + ResultSet rs = null; + ArrayList idList = new ArrayList<>(); + try { + ps = conn.prepareStatement(sql); + rs = ps.executeQuery(); + while (rs.next()) { + int userId = rs.getInt("submitter"); + idList.add(userId); + if (!userFirstVersion.containsKey(userId)) { + logger.warning( + "Vote table has user missing from users table, skipping user ID: " + + userId); + } else if (userFirstVersion.get(userId) == 0) { + userFirstVersion.put(userId, ver); + ++fixedCountAllVersions; + ++fixedCountThisVersion; + } + } + } finally { + DBUtils.close(ps, rs); + } + logger.warning( + "Got users from version " + + ver + + "; fixed (in this vote table): " + + fixedCountThisVersion + + "; so far fixed (in any vote table): " + + fixedCountAllVersions + + "; seconds elapsed = " + + (System.currentTimeMillis() - timeStart) / 1000); + Collections.sort(idList); + Collections.reverse(idList); + int idListSize = idList.size(); + if (idListSize > 10) { + idList.subList(10, idListSize).clear(); + } + verToLastUser.put(ver, idList.get(0)); + logger.warning("Highest IDs in version " + ver + " are: " + idList); + } + } + } + + /** + * Take advantage of the fact that user id is assigned sequentially in chronological order, to + * fill in the missing first-version (for users not found in the vote tables), and also to + * improve first-version that are out of order (for users who never voted or were added long + * before they made votes that are found in the vote tables). + * + * @param userFirstVersion map from user ID to version number, to be modified + * @param verToLastUser map from version number to the highest user ID that voted in that + * version + */ + private void improveFirstVersions( + TreeMap userFirstVersion, Map verToLastUser) { + conformToNeighbors(userFirstVersion); + improveByOrder(userFirstVersion); + flagByRanges(userFirstVersion, verToLastUser); + } + + private void improveByOrder(TreeMap userFirstVersion) { + int goodFirstVersion = 0; + for (int userId : userFirstVersion.descendingKeySet()) { + int ver = userFirstVersion.get(userId); + if (goodFirstVersion == 0) { + if (ver != 0) { + goodFirstVersion = ver; + } + } else if (ver == 0) { + logger.warning( + "improveByOrder: userId = " + + userId + + "; changing ver zero to " + + goodFirstVersion); + userFirstVersion.put(userId, goodFirstVersion); + } else if (ver > goodFirstVersion) { + logger.warning( + "improveByOrder: userId = " + + userId + + "; changing ver " + + ver + + " to " + + goodFirstVersion); + userFirstVersion.put(userId, goodFirstVersion); + } else if (ver < goodFirstVersion) { + long versionGap = goodFirstVersion - ver; + if (versionGap > 2) { + logger.warning( + "improveByOrder: userId = " + + userId + + "; NOT changing goodFirstVersion " + + goodFirstVersion + + " to " + + ver + + " since the gap is too large, versionGap = " + + versionGap); + } else { + logger.warning( + "improveByOrder: userId = " + + userId + + "; changing goodFirstVersion " + + goodFirstVersion + + " to " + + ver + + "; versionGap = " + + versionGap); + goodFirstVersion = ver; + } + } + } + } + + private void flagByRanges( + TreeMap userFirstVersion, Map verToLastUser) { + for (int userId : userFirstVersion.keySet()) { + int ver = userFirstVersion.get(userId); + if (ver == 0) { + continue; + } + if (!verToLastUser.containsKey(ver)) { + logger.warning("flagByRanges: version " + ver + " is missing from verToLastUser"); + continue; + } + int lastUserPerRange = verToLastUser.get(ver); + if (userId == lastUserPerRange + 1) { + verToLastUser.put(ver, userId); // OK, extend the range + } else if (userId > lastUserPerRange) { + logger.warning( + "flagByRanges: userId = " + + userId + + "; out of range for version " + + ver + + "; lastUserPerRange = " + + lastUserPerRange); + } + } + } + + /** + * Fix values that are too far removed from the mean of their close neighbors. For example, the + * user ID might imply the account was created in version 30, but the vote table might indicate + * the user voted in version 25. Such anomalies probably result from copying votes from one user + * to another. + * + * @param userFirstVersion map from user ID to version number, to be modified + */ + private void conformToNeighbors(TreeMap userFirstVersion) { + // Omit user IDs that map to zero, and separate the remainder into a + // list of users and a list of corresponding versions. + ArrayList denseUsers = new ArrayList<>(); + ArrayList denseVers = new ArrayList<>(); + for (int userId : userFirstVersion.keySet()) { + int ver = userFirstVersion.get(userId); + if (ver != 0) { + denseUsers.add(userId); + denseVers.add(ver); + } + } + int fixedCount = 0; + for (int i = 0; i < denseVers.size(); i++) { + int ver = denseVers.get(i); + int userId = denseUsers.get(i); + int fixedVer = fixAnomaly(denseVers, i); + if (fixedVer < ver) { + userFirstVersion.put(userId, fixedVer); + logger.warning( + "Anomaly fixed: userId = " + + userId + + "; ver = " + + ver + + " changed to " + + fixedVer); + ++fixedCount; + } else if (fixedVer > ver) { + logger.warning( + "Anomaly NOT fixed since fixed version would be later: userId = " + + userId + + "; ver = " + + ver + + " NOT changed to " + + fixedVer); + } + } + logger.warning("Total " + fixedCount + " anomalies fixed"); + } + + /** + * If the element at the specified index differs too much from the mean value of its neighbors, + * return that mean value. Otherwise, return the element unchanged. + * + * @param a the array + * @param i the index + * @return the possibly corrected value + */ + private int fixAnomaly(ArrayList a, int i) { + final int RANGE = 10; // how many neighbors to check on each side + int start = i - RANGE; + if (start < 0) { + start = 0; + } + int end = i + RANGE; + if (end > a.size() - 1) { + end = a.size() - 1; + } + ArrayList b = new ArrayList<>(); + for (int j = start; j < end; j++) { + if (j != i) { + b.add(a.get(j)); + } + } + int[] c = new int[b.size()]; + for (int j = 0; j < b.size(); j++) { + c[j] = b.get(j); + } + double m = median(c); + int val = a.get(i); + double delta = abs(m - val); + return (delta > 1.0) ? (int) Math.round(m) : val; + } + + static double median(int[] x) { + Arrays.sort(x); + int n = x.length; + if (n % 2 != 0) { + return x[n / 2]; + } else { + return (double) (x[(n - 1) / 2] + x[n / 2]) / 2.0; + } + } + + private Map mapVersionToFirstDate(Connection conn) throws SQLException { + // As of 2025-10, only these 19 versions have votes: + // 25, 26, 28, 30, 32, 33, 34, 35, 36, 37, 38, 40, 41, 42, 43, 44, 45, 46, 48 + final int firstVersion = SurveyAjax.oldestVersionForImportingVotes; + final int lastVersion = Integer.parseInt(SurveyMain.getNewVersion()); + Map verToDate = new TreeMap<>(); + for (int ver = firstVersion; ver <= lastVersion; ver++) { + String voteTable = + DBUtils.Table.VOTE_VALUE.forVersion(Integer.toString(ver), false).toString(); + if (DBUtils.hasTable(voteTable)) { + ResultSet rs = null; + PreparedStatement ps = null; + try { + final String sql = + "SELECT last_mod FROM " + voteTable + " ORDER BY last_mod LIMIT 1"; + ps = conn.prepareStatement(sql); + rs = ps.executeQuery(); + while (rs.next()) { + Timestamp date = rs.getTimestamp(1); + Timestamp adjustedDate = removeTime(date); + verToDate.put(ver, adjustedDate); + logger.warning( + "Version " + + ver + + " first date = " + + adjustedDate + + " (adjusted from " + + date + + ")"); + } + } finally { + DBUtils.close(ps, rs); + } + } + } + return verToDate; + } + + // Convert 2025-04-10 16:53:25 to 2025-04-10 00:00:00, for example + private static Timestamp removeTime(Timestamp timestamp) { + Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + cal.setTime(timestamp); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return new Timestamp(cal.getTimeInMillis()); + } + + /** + * Populate the firstdate column in the db based on the given maps + * + * @param conn the db connection + * @param userFirstVersion map from user ID to version number, already completed + * @param verToDate map from version number to date + * @throws SQLException if error + */ + private void populateFirstDateColumn( + Connection conn, + Map userFirstVersion, + Map verToDate) + throws SQLException { + final String sql = "UPDATE " + CLDR_USERS + " SET firstdate=? WHERE id=?"; + PreparedStatement ps = null; + try { + ps = conn.prepareStatement(sql); + for (int userId : userFirstVersion.keySet()) { + int ver = userFirstVersion.get(userId); + if (ver != 0) { + Timestamp firstdate = verToDate.get(ver); + ps.setTimestamp(1, firstdate); + ps.setInt(2, userId); + ps.executeUpdate(); + } + } + } finally { + DBUtils.close(ps); + } + } }