diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ccbc73..5b0706fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,11 +19,40 @@ ``` Versions released before this one stay available on Maven Central. The files are signed with the same key as before (`FC8DB0DE234A273BA45E562FB8C83A079A5BBD0C`). +* ! Minor breaking change ! SDK behavior settings are now enabled by default. The SDK fetches them from the server at initialization and every few hours, so changes made in SDK Manager > SDK Configuration on your server affect SDK behavior directly. They are not used in backend mode. +* ! Minor breaking change ! The feature interfaces of "Countly.instance()" ("events()", "views()", "crashes()", "location()", "userProfile()", "remoteConfig()", "feedback()", "content()" and "deviceId()") no longer return null. + Before initialization, while the consent of a feature is not given, or when a feature is not enabled, they return an interface that ignores calls until the feature runs and passes an error to the callbacks of the calls it ignores. + An interface you keep sends its calls to the feature once it runs. Use "Countly.isTracking" to check whether a feature runs. + +* Added support for SDK behavior settings that let the server control the SDK: + * Switches for tracking, networking, sessions, views, custom events, crash reporting, automatic crash reporting, location, and console logging + * The event queue size, request queue size, session update interval, and settings refresh interval + * The consent requirement, applied on the next initialization + * SDK internal limits for key length, value size, segmentation value count, stack trace lines per thread, stack trace line length, and breadcrumb count + * Listing filters (blacklist/whitelist) for events, segmentation keys, event specific segmentation keys, and user properties, plus a user property cache limit + * Entering, refreshing, and the fetch interval of the content zone, plus Journey Trigger Events and Journey Trigger Views that refresh it + * Dropping requests older than a given age +* Added "setSdkBehaviorSettings(String)" init config method for providing SDK behavior settings at initialization. +* Added "disableSdkBehaviorSettingsUpdates()" init config method for disabling SDK behavior settings requests. +* Improved request queue handling with a built-in backoff mechanism which is enabled by default. When the server takes 10 seconds or longer to answer while few requests are queued, the SDK pauses sending for 60 seconds. +* Added "disableBackoffMechanism()" init config method to disable the backoff behavior. +* Added support for server requested SDK log gathering. When the server asks for it, the SDK uploads its own internal log lines, even if logging is disabled. While consent is required, a line is uploaded only while the consent of the feature it belongs to is given. +* Added support for the SDK connection test. When the server asks for it, the SDK checks that it can reach every endpoint it depends on and reports the results. + +* Mitigated an issue where saving the SDK storage while remote config values were changing could empty the storage file. +* Mitigated an issue where a storage file cut short by a stopped process made initialization fail. +* Mitigated an issue where a request recorded while the SDK was finishing a send could wait until the next request or session update before it was sent. * Updated JSON library version from "20250517" to "20260814". * Mitigated an issue where requests, sessions or crashes created close together could get the same storage ID, so some of them were lost. +* Mitigated an issue where removing the consent of one feature also stopped the other features and deleted their queued events, unsent crashes and remote config values, after which the device ID and user profile interfaces could throw. +* Mitigated an issue where, while consent was required, unhandled crashes were not recorded if the crash reporting consent was given after initialization. +* Mitigated an issue where removing the location consent did not erase the location on the server. +* Mitigated an issue where an error in one feature during a timer tick stopped the SDK timer, and with it session updates and event sending. * Mitigated an issue where verbose logs showed the values of custom network request headers. Only the number of headers is logged now. +* Mitigated an issue where a fresh initialization logged an error for storage files that did not exist yet. +* Mitigated an issue where a crash breadcrumb added while a crash was being recorded on another thread could be lost. ## 26.8.0 * Added support for the Content feature, accessible through the "Countly.instance().content()" interface: diff --git a/sdk-java/src/main/java/ly/count/sdk/java/Config.java b/sdk-java/src/main/java/ly/count/sdk/java/Config.java index 56275a98..5cf1f4c2 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/Config.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/Config.java @@ -253,6 +253,26 @@ public class Config { protected boolean locationEnabled = true; protected boolean autoSendUserProperties = true; + /** + * Seed value for the SDK behavior settings. Used on first init only, + * when no settings have yet been persisted; subsequent inits use the + * value pulled from storage. + */ + protected String sdkBehaviorSettings = null; + + /** + * If {@code true}, the SDK skips the periodic {@code /o/sdk?method=sc} + * fetch. The seed JSON from {@link #setSdkBehaviorSettings(String)} (or + * a previously-persisted snapshot) is still applied. + */ + protected boolean sdkBehaviorSettingsRequestsDisabled = false; + + /** + * Whether the backoff mechanism may hold the request queue back after the server answered + * slowly. The SDK behavior settings can override it. + */ + protected boolean backoffMechanismEnabled = true; + // TODO: storage limits & configuration // protected int maxRequestsStored = 0; // protected int storageDirectory = ""; @@ -1508,4 +1528,46 @@ public Config disableAutoSendUserProperties() { this.autoSendUserProperties = false; return this; } + + /** + * Provide an initial SDK behavior settings payload to apply at init time, + * before the first {@code /o/sdk?method=sc} fetch returns. The value is + * the same JSON shape the Countly server emits: {@code {"v":1,"t":, + * "c":{...}}}. The seed is only used when nothing has been persisted yet, + * and it stands in for the server's settings: the first answer from the + * server replaces it, so a setting the answer does not carry goes back to + * this configuration. A {@code "cr":true} here can be lifted by that answer; + * use {@link #setRequiresConsent(boolean)} for a requirement that must hold. + * + * @param sdkBehaviorSettings serialized config JSON, or {@code null} for none + * @return {@code this} instance for method chaining + */ + public Config setSdkBehaviorSettings(String sdkBehaviorSettings) { + this.sdkBehaviorSettings = sdkBehaviorSettings; + return this; + } + + /** + * Disable the periodic SDK behavior settings fetch from the server. Any + * seed JSON or previously-persisted snapshot is still applied at init. + * + * @return {@code this} instance for method chaining + */ + public Config disableSdkBehaviorSettingsUpdates() { + this.sdkBehaviorSettingsRequestsDisabled = true; + return this; + } + + /** + * Disable the backoff mechanism. While it is on, a request the server takes long to answer + * makes the SDK wait before it sends the next queued request, which gives a busy server room + * to recover. It never waits while many requests are queued or after an old request. The SDK + * behavior settings can still turn it on or off. + * + * @return {@code this} instance for method chaining + */ + public Config disableBackoffMechanism() { + this.backoffMechanismEnabled = false; + return this; + } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/Countly.java b/sdk-java/src/main/java/ly/count/sdk/java/Countly.java index 86486768..30f5bf9c 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/Countly.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/Countly.java @@ -84,7 +84,6 @@ public void init(final Config config) { } Log L = new Log(internalConfig.loggingLevel, internalConfig.logListener); - internalConfig.setLogger(L); if (directory == null) { L.e("[Countly] File cannot be null"); @@ -104,6 +103,7 @@ public void init(final Config config) { L.e("[Countly] Countly shouldn't be initialized twice. Please either use Countly.isInitialized() to check status or call Countly.stop() before second Countly.init(). Calling 'init' the second time will now do nothing"); return; } + internalConfig.setLogger(L); if (internalConfig.enableBackendMode) { internalConfig.sdkName = "java-native-backend"; @@ -396,14 +396,15 @@ public static void onConsentRemoval(Config.Feature... features) { * Feedback interface to use feedback widgets nps,rating and forms. * And do manual reporting of feedbacks. * - * @return {@link ModuleFeedback.Feedback} instance. + * @return {@link ModuleFeedback.Feedback} instance, never {@code null}: before init, or while the feature does not + * run, as its consent is not given or it is not enabled, one that ignores calls until it runs */ public ModuleFeedback.Feedback feedback() { if (!isInitialized()) { if (L != null) { L.e("[Countly] SDK is not initialized yet."); } - return null; + return SDKCore.dormantFeedback; } return sdk.feedback(); } @@ -413,7 +414,8 @@ public ModuleFeedback.Feedback feedback() { * content block. A {@link ly.count.sdk.java.internal.ContentDisplay} has to be registered * before a content zone can be entered; the "ly.count.sdk:java-ui" artifact ships a JavaFX one. * - * @return {@link ModuleContent.Content} instance. + * @return {@link ModuleContent.Content} instance, never {@code null}: before init, or while the feature does not + * run, as its consent is not given or it is not enabled, one that ignores calls until it runs * @apiNote This is an EXPERIMENTAL feature, and it can have breaking changes */ public ModuleContent.Content content() { @@ -421,7 +423,7 @@ public ModuleContent.Content content() { if (L != null) { L.e("[Countly] content, SDK is not initialized yet."); } - return null; + return SDKCore.dormantContent; } return sdk.content(); } @@ -429,14 +431,15 @@ public ModuleContent.Content content() { /** * RemoteConfig interface to use remote config feature. * - * @return {@link ModuleRemoteConfig.RemoteConfig} instance. + * @return {@link ModuleRemoteConfig.RemoteConfig} instance, never {@code null}: before init, or while the feature does not + * run, as its consent is not given or it is not enabled, one that ignores calls until it runs */ public ModuleRemoteConfig.RemoteConfig remoteConfig() { if (!isInitialized()) { if (L != null) { L.e("[Countly] remoteConfig, SDK is not initialized yet."); } - return null; + return SDKCore.dormantRemoteConfig; } return sdk.remoteConfig(); } @@ -444,14 +447,15 @@ public ModuleRemoteConfig.RemoteConfig remoteConfig() { /** * DeviceId interface to use device id functionalities. * - * @return {@link ModuleDeviceIdCore.DeviceId} instance. + * @return {@link ModuleDeviceIdCore.DeviceId} instance, never {@code null}: before init, or while the feature does not + * run, as its consent is not given or it is not enabled, one that ignores calls until it runs */ public ModuleDeviceIdCore.DeviceId deviceId() { if (!isInitialized()) { if (L != null) { L.e("[Countly] SDK is not initialized yet."); } - return null; + return SDKCore.dormantDeviceId; } return sdk.deviceId(); } @@ -472,14 +476,15 @@ public Event event(String key) { /** * Event module calls * - * @return event module otherwise null if SDK is not initialized + * @return {@link ModuleEvents.Events} instance, never {@code null}: before init, or while the feature does not + * run, as its consent is not given or it is not enabled, one that ignores calls until it runs */ public ModuleEvents.Events events() { if (!isInitialized()) { if (L != null) { L.e("[Countly] SDK is not initialized yet."); } - return null; + return SDKCore.dormantEvents; } return sdk.events(); } @@ -487,14 +492,15 @@ public ModuleEvents.Events events() { /** * Crash module calls * - * @return crash module otherwise null if SDK is not initialized + * @return {@link ModuleCrashes.Crashes} instance, never {@code null}: before init, or while the feature does not + * run, as its consent is not given or it is not enabled, one that ignores calls until it runs */ public ModuleCrashes.Crashes crashes() { if (!isInitialized()) { if (L != null) { L.e("[Countly] SDK is not initialized yet."); } - return null; + return SDKCore.dormantCrashes; } return sdk.crashes(); } @@ -502,14 +508,15 @@ public ModuleCrashes.Crashes crashes() { /** * Views interface to use views feature. * - * @return {@link ModuleViews.Views} instance. + * @return {@link ModuleViews.Views} instance, never {@code null}: before init, or while the feature does not + * run, as its consent is not given or it is not enabled, one that ignores calls until it runs */ public ModuleViews.Views views() { if (!isInitialized()) { if (L != null) { L.e("[Countly] SDK is not initialized yet."); } - return null; + return SDKCore.dormantViews; } return sdk.views(); } @@ -530,14 +537,15 @@ public Event timedEvent(String key) { /** * UserProfile interface to use user profile feature. * - * @return {@link ModuleUserProfile.UserProfile} instance. + * @return {@link ModuleUserProfile.UserProfile} instance, never {@code null}: before init, or while the feature does not + * run, as its consent is not given or it is not enabled, one that ignores calls until it runs */ public ModuleUserProfile.UserProfile userProfile() { if (!isInitialized()) { if (L != null) { L.e("[Countly] userProfile, SDK is not initialized yet."); } - return null; + return SDKCore.dormantUserProfile; } return sdk.userProfile(); } @@ -545,7 +553,8 @@ public ModuleUserProfile.UserProfile userProfile() { /** * Location interface to use location feature. * - * @return {@link ModuleLocation.Location} instance. + * @return {@link ModuleLocation.Location} instance, never {@code null}: before init, or while the feature does not + * run, as its consent is not given or it is not enabled, one that ignores calls until it runs */ public ModuleLocation.Location location() { if (!isInitialized()) { @@ -553,7 +562,7 @@ public ModuleLocation.Location location() { if (L != null) { L.e("Countly.sharedInstance().init must be called before accessing location"); } - return null; + return SDKCore.dormantLocation; } return sdk.location(); @@ -574,25 +583,25 @@ public User user() { @Override public Usage addParam(String key, Object value) { - L.d("[Countly] addParam: key = " + key + " value = " + value); + L.d("[Countly] addParam: key = " + key); return ((Session) sdk.session(null)).addParam(key, value); } @Override public Usage addCrashReport(Throwable t, boolean fatal) { - L.d("[Countly] addCrashReport: t = " + t + " fatal = " + fatal); + L.d("[Countly] addCrashReport: fatal = " + fatal); return ((Session) sdk.session(null)).addCrashReport(t, fatal); } @Override public Usage addCrashReport(Throwable t, boolean fatal, String name, Map segments, String... logs) { - L.d("[Countly] addCrashReport: t = " + t + " fatal = " + fatal + " name = " + name + " segments = " + segments + " logs = " + logs); + L.d("[Countly] addCrashReport: fatal = " + fatal); return ((Session) sdk.session(null)).addCrashReport(t, fatal, name, segments, logs); } @Override public Usage addLocation(double latitude, double longitude) { - L.d("[Countly] addLocation: latitude = " + latitude + " longitude = " + longitude); + L.d("[Countly] addLocation"); return ((Session) sdk.session(null)).addLocation(latitude, longitude); } @@ -603,7 +612,7 @@ public Usage addLocation(double latitude, double longitude) { */ @Override public View view(String name, boolean start) { - L.d("[Countly] view: name = " + name + " start = " + start); + L.d("[Countly] view: start = " + start); return ((Session) sdk.session(null)).view(name, start); } @@ -614,7 +623,7 @@ public View view(String name, boolean start) { */ @Override public View view(String name) { - L.d("[Countly] view: name = " + name); + L.d("[Countly] view"); return ((Session) sdk.session(null)).view(name); } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java new file mode 100644 index 00000000..43356e0d --- /dev/null +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java @@ -0,0 +1,425 @@ +package ly.count.sdk.java.internal; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * The SDK behavior settings in effect, the server side configuration served by + * {@code /o/sdk?method=sc}. Every value starts from the developer configuration and is overridden, + * in increasing order of precedence, by the settings provided at init (used only while nothing is + * stored), by the settings stored from an earlier run and by the latest server response. + *

+ * {@link ModuleConfiguration} implements it for the running SDK. Reach it through + * {@link InternalConfig#getConfigurationProvider()}, which never returns {@code null}. A server + * response can change any value on the thread that delivered it, so read a value where it is used + * instead of keeping a copy of it. + */ +public interface ConfigurationProvider { + + /** + * Key {@code networking}. + * + * @return whether requests may be sent at all, the queued ones and the immediate ones + */ + boolean getNetworkingEnabled(); + + /** + * Key {@code tracking}. + * + * @return whether anything new may be recorded + */ + boolean getTrackingEnabled(); + + /** + * Key {@code st}. + * + * @return whether sessions may be tracked + */ + boolean getSessionTrackingEnabled(); + + /** + * Key {@code vt}. + * + * @return whether views may be tracked + */ + boolean getViewTrackingEnabled(); + + /** + * Key {@code cet}. + * + * @return whether custom events, the ones whose key is not one of {@link ModuleEvents#INTERNAL_EVENT_KEYS}, may be recorded + */ + boolean getCustomEventTrackingEnabled(); + + /** + * Key {@code ecz}. + * + * @return whether the SDK should be in the content zone + */ + boolean getContentZoneEnabled(); + + /** + * Key {@code crt}. + * + * @return whether crashes may be recorded + */ + boolean getCrashReportingEnabled(); + + /** + * Key {@code ast}. This SDK has no automatic session tracking, so the value is only exposed. + * + * @return whether sessions should be tracked automatically + */ + boolean getAutomaticSessionTrackingEnabled(); + + /** + * Key {@code avt}. This SDK has no automatic view tracking, so the value is only exposed. + * + * @return whether views should be tracked automatically + */ + boolean getAutomaticViewTrackingEnabled(); + + /** + * Key {@code acr}. + * + * @return whether unhandled exceptions should be reported automatically + */ + boolean getAutomaticCrashReportingEnabled(); + + /** + * Key {@code lt}. + * + * @return whether location may be tracked + */ + boolean getLocationTrackingEnabled(); + + /** + * Key {@code rcz}. + * + * @return whether the content zone may be refreshed + */ + boolean getRefreshContentZoneEnabled(); + + /** + * Key {@code bom}. + * + * @return whether the backoff mechanism is on + */ + boolean getBOMEnabled(); + + /** + * Key {@code log}. + * + * @return whether the SDK should print its logs + */ + boolean getLoggingEnabled(); + + /** + * Key {@code cr}. The SDK applies it when it initializes, so a value that arrives later takes + * effect on the next init and can differ from {@link InternalConfig#requiresConsent()} until then. + * + * @return whether consent is required before anything is recorded + */ + boolean getConsentRequired(); + + /** + * Key {@code scui}. + * + * @return hours between two fetches of the settings + */ + int getServerConfigUpdateInterval(); + + /** + * Key {@code rqs}. + * + * @return the maximum number of queued requests + */ + int getRequestQueueMaxSize(); + + /** + * Whether {@link #getRequestQueueMaxSize()} comes from the settings, provided, stored or + * received, rather than from the developer configuration. + * + * @return {@code true} when the settings set the request queue size + */ + boolean isRequestQueueMaxSizeFromBehaviorSettings(); + + /** + * Key {@code eqs}. + * + * @return how many events are held before they are sent + */ + int getEventQueueSizeThreshold(); + + /** + * Key {@code sui}. + * + * @return seconds between two ticks of the SDK timer, which sends session updates and queued events + */ + int getSessionUpdateInterval(); + + /** + * Key {@code lkl}. + * + * @return the maximum length of a key, {@link Integer#MAX_VALUE} for no limit + */ + int getMaxKeyLength(); + + /** + * Key {@code lvs}. + * + * @return the maximum length of a value, {@link Integer#MAX_VALUE} for no limit + */ + int getMaxValueSize(); + + /** + * Key {@code lsv}. + * + * @return the maximum number of segmentation entries, {@link Integer#MAX_VALUE} for no limit + */ + int getMaxSegmentationValues(); + + /** + * Key {@code lbc}. + * + * @return the maximum number of breadcrumbs kept for a crash + */ + int getMaxBreadcrumbCount(); + + /** + * Key {@code ltlpt}. + * + * @return the maximum number of stack trace lines per thread, {@link Integer#MAX_VALUE} for no limit + */ + int getMaxStackTraceLinesPerThread(); + + /** + * Key {@code ltl}. + * + * @return the maximum length of a stack trace line, {@link Integer#MAX_VALUE} for no limit + */ + int getMaxStackTraceLineLength(); + + /** + * Key {@code upcl}. + * + * @return the maximum number of custom user properties cached before they are sent, + * {@link Integer#MAX_VALUE} for no limit + */ + int getUserPropertyCacheLimit(); + + /** + * Key {@code bom_at}. + * + * @return seconds a request may take before the backoff mechanism counts it as slow + */ + int getBOMAcceptedTimeoutSeconds(); + + /** + * Key {@code bom_rqp}. + * + * @return the share of the request queue, in (0, 1), above which the backoff mechanism stays off + */ + double getBOMRQPercentage(); + + /** + * Key {@code bom_ra}. + * + * @return the age, in hours, above which a request is too old for the backoff mechanism to delay it + */ + int getBOMRequestAge(); + + /** + * Key {@code bom_d}. + * + * @return seconds the backoff mechanism holds the request queue back + */ + int getBOMDuration(); + + /** + * Key {@code dort}. + * + * @return the age, in hours, above which a queued request is dropped, {@code 0} to keep every request + */ + int getRequestDropAgeHours(); + + /** + * Key {@code czi}. + * + * @return seconds between two content fetches while in the content zone + */ + int getContentZoneTimerInterval(); + + /** + * Keys {@code eb} and {@code ew}. + * + * @return the filter for custom event keys, never {@code null} + */ + @Nonnull FilterList> getEventFilterList(); + + /** + * Keys {@code upb} and {@code upw}. + * + * @return the filter for custom user property keys, never {@code null} + */ + @Nonnull FilterList> getUserPropertyFilterList(); + + /** + * Keys {@code sb} and {@code sw}. + * + * @return the filter for custom event segmentation keys, never {@code null} + */ + @Nonnull FilterList> getSegmentationFilterList(); + + /** + * Keys {@code esb} and {@code esw}. + * + * @return the segmentation key filter of each custom event key, never {@code null} + */ + @Nonnull FilterList>> getEventSegmentationFilterList(); + + /** + * Key {@code jte}. + * + * @return the custom event keys that trigger a journey, never {@code null} and never modifiable + */ + @Nonnull Set getJourneyTriggerEvents(); + + /** + * Key {@code jtv}. + * + * @return the view names that trigger a journey, never {@code null} and never modifiable + */ + @Nonnull Set getJourneyTriggerViews(); + + /** + * The decision of the {@code lg} directive, which sits next to {@code c} in a live response. + * + * @return whether this device gathers its SDK logs for the server + */ + @Nonnull LogGatheringState getLogGatheringState(); + + /** + * The gather id every uploaded log batch has to carry. + * + * @return the id, {@code null} unless {@link #getLogGatheringState()} is {@link LogGatheringState#GATHERING} + */ + @Nullable String getLogGatheringId(); + + /** + * The log levels to gather, as level characters: e error, w warning, i info, d debug, v verbose. + * + * @return a non empty subset of {@code ewidv} + */ + @Nonnull String getLogGatheringLevels(); + + /** + * How many log lines to hold before a batch is uploaded. + * + * @return the batch size, within [10, 500] + */ + int getLogGatheringBatchSize(); + + /** + * Whether this device gathers its SDK logs for the server. + */ + enum LogGatheringState { + /** + * No live response has decided yet: only stored or provided settings were seen. + */ + UNDECIDED, + /** + * A live response enabled gathering and carried a usable gather id. + */ + GATHERING, + /** + * A live response, a failed fetch or disabled settings requests decided against gathering. + */ + NOT_GATHERING + } + + /** + * An immutable listing filter: names that are the only ones allowed, a whitelist, or the ones + * rejected, a blacklist. An empty blacklist allows everything. + * + * @param the names, as a set or as a set per event key + */ + final class FilterList { + static final FilterList> NO_NAMES = new FilterList<>(Collections.emptySet(), false); + static final FilterList>> NO_NAMES_PER_EVENT = new FilterList<>(Collections.>emptyMap(), false); + + private final T names; + private final boolean isWhitelist; + + /** + * Wraps names that are already immutable. + * + * @param names the names + * @param isWhitelist {@code true} when only the names are allowed, {@code false} when they are rejected + */ + private FilterList(@Nonnull T names, boolean isWhitelist) { + this.names = names; + this.isWhitelist = isWhitelist; + } + + /** + * Builds a filter over an immutable copy of the given names. + * + * @param names the names, kept in the given order + * @param isWhitelist {@code true} when only the names are allowed, {@code false} when they are rejected + * @return the filter + */ + static FilterList> ofNames(@Nonnull Set names, boolean isWhitelist) { + return new FilterList<>(Collections.unmodifiableSet(new LinkedHashSet<>(names)), isWhitelist); + } + + /** + * Builds a filter over an immutable copy of the given names of each event key. + * + * @param namesPerEvent the names of each event key, kept in the given order + * @param isWhitelist {@code true} when only the names are allowed, {@code false} when they are rejected + * @return the filter + */ + static FilterList>> ofNamesPerEvent(@Nonnull Map> namesPerEvent, boolean isWhitelist) { + Map> copy = new LinkedHashMap<>(); + for (Map.Entry> entry : namesPerEvent.entrySet()) { + copy.put(entry.getKey(), Collections.unmodifiableSet(new LinkedHashSet<>(entry.getValue()))); + } + return new FilterList<>(Collections.unmodifiableMap(copy), isWhitelist); + } + + /** + * The names of the filter. + * + * @return the names, never modifiable + */ + @Nonnull + public T getFilterList() { + return names; + } + + /** + * Whether the names are the only ones allowed. + * + * @return {@code true} for a whitelist, {@code false} for a blacklist + */ + public boolean isWhitelist() { + return isWhitelist; + } + + /** + * Describes the filter for logs. + * + * @return the kind of the filter followed by its names + */ + @Override + public String toString() { + return (isWhitelist ? "whitelist " : "blacklist ") + names; + } + } +} diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/CountlyTimer.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/CountlyTimer.java index f2dd2205..a79481d7 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/CountlyTimer.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/CountlyTimer.java @@ -8,6 +8,7 @@ public class CountlyTimer { private final Log L; private ScheduledExecutorService timerService; + private volatile long timerDelaySeconds = -1; protected static int TIMER_DELAY_MS = 0; // for testing purposes protected CountlyTimer(Log logger) { @@ -53,6 +54,7 @@ protected void startTimer(long timerDelay, Runnable runnable) { */ protected void startTimer(long timerDelay, long initialDelayMs, Runnable runnable) { L.i("[CountlyTimer] startTimer, Starting global timer timerDelay: [" + timerDelay + "] initialDelayMs: [" + initialDelayMs + "]"); + timerDelaySeconds = timerDelay; long delay = timerDelay * 1000; if (delay < 1000) { @@ -66,6 +68,22 @@ protected void startTimer(long timerDelay, long initialDelayMs, Runnable runnabl startTime = 0; } - timerService.scheduleWithFixedDelay(runnable, startTime, delay, TimeUnit.MILLISECONDS); + timerService.scheduleWithFixedDelay(() -> { + try { + runnable.run(); + } catch (Throwable t) { + //a fixed delay schedule ends for good once a run throws + L.e("[CountlyTimer] a timer run failed, [" + t + "]"); + } + }, startTime, delay, TimeUnit.MILLISECONDS); + } + + /** + * The interval this timer was last started with, as it was requested. + * + * @return the interval in seconds, {@code -1} while the timer was never started + */ + long getTimerDelaySeconds() { + return timerDelaySeconds; } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/CrashImpl.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/CrashImpl.java index a3b0ff9d..20665634 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/CrashImpl.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/CrashImpl.java @@ -6,9 +6,11 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nonnull; import ly.count.sdk.java.Crash; import org.json.JSONException; import org.json.JSONObject; @@ -23,6 +25,10 @@ public class CrashImpl implements Crash, Storable { private final JSONObject data; private Throwable throwable; private Map traces; + private Thread tracesMainThread; + //whether "_error" holds the dump of addTraces rather than the stack trace of a throwable + private boolean errorIsTraceDump = false; + private String[] logs; protected CrashImpl(Log logger) { this(TimeUtils.uniqueStorageId(), logger); @@ -38,6 +44,7 @@ protected CrashImpl(Long id, Log logger) { @Override public CrashImpl addThrowable(Throwable throwable) { this.throwable = throwable; + this.errorIsTraceDump = false; StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); @@ -57,31 +64,58 @@ public CrashImpl addTraces(Thread main, Map traces) return this; } else { this.traces = traces; + String dump = printAllTraces(main, traces, Integer.MAX_VALUE, "[CrashImpl] addTraces"); + this.tracesMainThread = main; + this.errorIsTraceDump = true; + return add("_type", "anr").add("_error", dump); + } + } - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); + /** + * Prints the stack traces of every thread, the main thread first. + * + * @param main the main thread, {@code null} for none + * @param threadTraces the stack trace of each thread + * @param maxLinesPerThread how many of the top lines of each thread to print at most + * @param tag the caller, for the log of a thread that has more lines + * @return the printed stack traces + */ + private String printAllTraces(Thread main, @Nonnull Map threadTraces, int maxLinesPerThread, @Nonnull String tag) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); - if (main != null && traces.containsKey(main)) { - pw.println("Thread [main]:"); - printTraces(pw, null, traces.get(main)); - pw.append("\n\n"); - } + if (main != null && threadTraces.containsKey(main)) { + pw.println("Thread [main]:"); + printTraces(pw, main, true, threadTraces.get(main), maxLinesPerThread, tag); + pw.append("\n\n"); + } - for (Thread thread : traces.keySet()) { - if (thread != main) { - printTraces(pw, thread, traces.get(thread)); - pw.append("\n\n"); - } + for (Thread thread : threadTraces.keySet()) { + if (thread != main) { + printTraces(pw, thread, false, threadTraces.get(thread), maxLinesPerThread, tag); + pw.append("\n\n"); } - return add("_type", "anr").add("_error", sw.toString()); } + return sw.toString(); } - private void printTraces(PrintWriter pw, Thread thread, StackTraceElement[] traces) { - if (thread != null) { + /** + * Prints the stack trace of one thread, under a header naming the thread unless it is the main one. + * + * @param pw where to print + * @param thread the thread + * @param mainThread whether it is the main thread, whose header is printed by the caller + * @param traces the stack trace of the thread + * @param maxLines how many of the top lines to print at most + * @param tag the caller, for the log when lines are left out + */ + private void printTraces(PrintWriter pw, Thread thread, boolean mainThread, StackTraceElement[] traces, int maxLines, @Nonnull String tag) { + if (!mainThread && thread != null) { pw.append("Thread [").append(thread.getName()).append("]:\n"); } - for (StackTraceElement el : traces) { + int lines = UtilsInternalLimits.stackTraceLinesToKeep(traces.length, maxLines, thread == null ? null : thread.getName(), L, tag); + for (int i = 0; i < lines; i++) { + StackTraceElement el = traces[i]; pw.append("\tat ").append(el == null ? "<>" : el.toString()).append("\n"); } } @@ -111,11 +145,90 @@ protected CrashImpl addSegments(Map segments) { @Override public CrashImpl setLogs(String[] logs) { if (logs != null && logs.length > 0) { + this.logs = logs.clone(); return add("_logs", Utils.join(Arrays.asList(logs), "\n")); } return this; } + /** + * Applies the SDK internal limits of the SDK behavior settings to what this crash sends: the lines + * per thread of a dump made by {@link #addTraces(Thread, Map)}, the length of every stack trace + * line, the keys, string values and number of the custom segments and the length of every + * breadcrumb. The stack trace of a throwable keeps all its lines. Nothing changes while no limit + * is exceeded. + * + * @param limits the settings in effect + * @param tag the caller, for the log + */ + void applyInternalLimits(@Nonnull ConfigurationProvider limits, @Nonnull String tag) { + int maxLinesPerThread = limits.getMaxStackTraceLinesPerThread(); + if (errorIsTraceDump && traceDumpExceeds(maxLinesPerThread)) { + add("_error", printAllTraces(tracesMainThread, traces, maxLinesPerThread, tag)); + } + + String error = data.optString("_error", null); + if (error != null) { + String truncatedError = UtilsInternalLimits.truncateStackTraceLines(error, limits.getMaxStackTraceLineLength(), L, tag); + if (truncatedError.length() != error.length()) { + add("_error", truncatedError); + } + } + + JSONObject custom = data.optJSONObject("_custom"); + if (custom != null) { + Map segments = new LinkedHashMap<>(); + for (String key : custom.keySet()) { + segments.put(key, custom.opt(key)); + } + Map limitedSegments = UtilsInternalLimits.applySegmentationLimits(segments, limits, L, tag); + if (!segments.equals(limitedSegments)) { + add("_custom", new JSONObject(limitedSegments)); + } + } + + applyValueSizeLimitToLogs(limits.getMaxValueSize(), tag); + } + + /** + * Whether a thread of the dump made by {@link #addTraces(Thread, Map)} has more stack trace lines + * than a limit. + * + * @param maxLinesPerThread the limit + * @return {@code true} when a thread has more lines + */ + private boolean traceDumpExceeds(int maxLinesPerThread) { + for (StackTraceElement[] threadTraces : traces.values()) { + if (threadTraces != null && threadTraces.length > maxLinesPerThread) { + return true; + } + } + return false; + } + + /** + * Cuts every breadcrumb given to {@link #setLogs(String[])} to the value size limit. + * + * @param maxValueSize the limit + * @param tag the caller, for the log + */ + private void applyValueSizeLimitToLogs(int maxValueSize, @Nonnull String tag) { + if (logs == null) { + return; + } + + String[] truncatedLogs = new String[logs.length]; + boolean truncated = false; + for (int i = 0; i < logs.length; i++) { + truncatedLogs[i] = UtilsInternalLimits.truncateValue(logs[i], maxValueSize, L, tag); + truncated |= logs[i] != null && truncatedLogs[i].length() != logs[i].length(); + } + + if (truncated) { + setLogs(truncatedLogs); + } + } + @Override public Throwable getThrowable() { return throwable; diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/DefaultNetworking.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/DefaultNetworking.java index 085775c3..10b00aab 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/DefaultNetworking.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/DefaultNetworking.java @@ -1,20 +1,57 @@ package ly.count.sdk.java.internal; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + public class DefaultNetworking implements Networking { + /** + * Name of the thread that ends a backoff of the request queue. + */ + static final String BACKOFF_THREAD_NAME = "network-backoff"; + private static final String TIMESTAMP_PARAM = "timestamp"; + private static final long MS_IN_HOUR = 60L * 60L * 1000L; + private Log L = null; private Transport transport; private Tasks tasks; - private boolean shutdown; + private volatile boolean shutdown; + private volatile boolean backedOff; + + private final Object recheckLock = new Object(); + //a check asked for while a task ran, which a drain that found the queue empty would otherwise lose: + //null for none, else whether every check asked for came from the log gathering's own transport work + private Boolean pendingRecheck = null; + private boolean backendMode; + private ScheduledExecutorService backoffScheduler; IStorageForRequestQueue storageForRequestQueue; + /** + * Prepares the transport, the thread requests are sent on and the scheduler that ends a + * backoff, which starts its daemon thread only when a first backoff is scheduled. + * + * @param config configuration of the SDK being initialized + * @param storageForRequestQueue the request queue to drain + */ @Override public void init(InternalConfig config, IStorageForRequestQueue storageForRequestQueue) { L = config.getLogger(); shutdown = false; + backedOff = false; + backendMode = config.isBackendModeEnabled(); transport = new Transport(); transport.init(config); tasks = new Tasks("network", L); + backoffScheduler = Executors.newSingleThreadScheduledExecutor(runnable -> { + Thread thread = new Thread(runnable, BACKOFF_THREAD_NAME); + thread.setDaemon(true); + return thread; + }); this.storageForRequestQueue = storageForRequestQueue; } @@ -23,54 +60,411 @@ public boolean isSending() { return tasks.isRunning(); } + /** + * Starts draining the request queue unless the SDK stopped, there is no device ID yet or the + * backoff mechanism holds the queue back. While a request is being handled the check is kept + * for when the drain ends without handing a request over. A drain started from a thread marked + * as doing the log gathering's own transport work, which queued or sent a batch of gathered log + * lines, keeps that mark until it read the request to send. + * + * @param config configuration of the running SDK + * @return whether a request is being handled + */ @Override public boolean check(InternalConfig config) { - L.d("[Networking] [check] state: shutdown [" + shutdown + "], tasks running [" + tasks.isRunning() + "], net running [" + tasks.isRunning() + "], device id [" + config.getDeviceId() + "]"); - if (!shutdown && !tasks.isRunning() && config.getDeviceId() != null) { - tasks.run(submit(config)); + L.d("[Networking] [check] state: shutdown [" + shutdown + "], backed off [" + backedOff + "], tasks running [" + tasks.isRunning() + "], net running [" + tasks.isRunning() + "], device id [" + config.getDeviceId() + "]"); + if (!shutdown && !backedOff && config.getDeviceId() != null) { + boolean ownTransportWork = L.isOwnTransportWork(); + boolean drainNow; + //decided under the lock the end of a drain takes, so a check kept for that end is never kept too late + synchronized (recheckLock) { + drainNow = !tasks.isRunning(); + if (!drainNow) { + pendingRecheck = pendingRecheck == null ? ownTransportWork : pendingRecheck && ownTransportWork; + } + } + if (drainNow) { + drain(config, ownTransportWork); + } } return tasks.isRunning(); } - protected Tasks.Task submit(final InternalConfig config) { + /** + * Starts a queue drain. When it ends without handing a request over for sending, failing + * included, it checks again if a check was asked for meanwhile, as a request stored just after the drain read the + * queue would otherwise wait for the next check. A drain that hands a request over is followed + * by the check of that send, so one request is handled at a time. + * + * @param config configuration of the running SDK + * @param followsOwnTransportWork whether the log gathering's own transport work started the drain + */ + private void drain(@Nonnull InternalConfig config, boolean followsOwnTransportWork) { + synchronized (recheckLock) { + pendingRecheck = null; + } + final Tasks.Task drain = submit(config, followsOwnTransportWork); + tasks.run(new Tasks.Task(drain.id) { + /** + * Runs the drain, a failure counting as a drain that handed nothing over. + */ + @Override + public Boolean call() { + try { + return drain.call(); + } catch (Exception e) { + L.e("[Networking] drain, failed to handle the next request, [" + e + "]"); + return false; + } + } + }, handedOver -> { + if (!Boolean.TRUE.equals(handedOver)) { + recheckIfRequested(config); + } + }); + } + + /** + * Runs a check kept while a task ran, marked as the log gathering's own transport work when + * every thread that asked for it was. + * + * @param config configuration of the running SDK + */ + private void recheckIfRequested(@Nonnull InternalConfig config) { + Boolean followsOwnTransportWork; + synchronized (recheckLock) { + followsOwnTransportWork = pendingRecheck; + pendingRecheck = null; + } + if (followsOwnTransportWork == null) { + return; + } + + L.setOwnTransportWork(followsOwnTransportWork); + try { + check(config); + } finally { + L.setOwnTransportWork(false); + } + } + + /** + * Sends the oldest queued request, unless networking is off or the backoff mechanism holds the + * queue back. Queued requests older than the request drop age are dropped first, unsent. Once + * the server accepts the request the next one follows, unless the server answered so slowly + * that the backoff mechanism holds the queue back. + * + * @param config configuration of the running SDK + * @param followsOwnTransportWork whether the log gathering's own transport work started the + * drain, which then keeps the thread marked while the drain checks whether it may run and + * reads the request, so a drain that follows an upload does not report on it + * @return the task, resolving to whether a request was handed over for sending or removed + */ + protected Tasks.Task submit(final InternalConfig config, final boolean followsOwnTransportWork) { return new Tasks.Task(Tasks.ID_STRICT) { @Override public Boolean call() throws Exception { - final Request request = storageForRequestQueue.getNextRequest(); - if (request == null) { - return false; - } else { - L.d("[Networking] Preparing request: " + request); - final Boolean check = SDKCore.instance.isRequestReady(request); - if (check == null) { - L.d("[Networking] Request is not ready yet: " + request); + L.setOwnTransportWork(followsOwnTransportWork); + try { + if (!config.getNetworkingEnabled()) { + L.d("[Networking] submit, networking disabled by SDK behavior settings; skipping queue drain"); return false; - } else if (check.equals(Boolean.FALSE)) { - L.d("[Networking] Request won't be ready, removing: " + request); - Storage.remove(config, request); - return true; - } else { - if (request.params.has("rr")) { - request.params.remove("rr"); - } - request.params.add("rr", storageForRequestQueue.remaningRequests()); - tasks.run(transport.send(request), result -> { - L.d("[Networking] Request " + request.storageId() + " sent?: " + result); - if (result) { - storageForRequestQueue.removeRequest(request); - check(config); - } - }); - return true; } + if (backedOff) { + L.d("[Networking] submit, the backoff mechanism holds the request queue back; skipping queue drain"); + return false; + } + } finally { + L.setOwnTransportWork(false); + } + final Request request = nextRequestWithinDropAge(config, followsOwnTransportWork); + if (request == null) { + return false; + } + + boolean uploadsGatheredLogs = request.params.has(Log.keySdkLogs); + L.setOwnTransportWork(uploadsGatheredLogs); + try { + return prepareAndSend(config, request, uploadsGatheredLogs); + } finally { + L.setOwnTransportWork(false); } } }; } + /** + * Hands a queued request to the transport once its owner says it is ready, or removes it when + * its owner says it never will be. While the request carries a batch of gathered log lines, the + * thread is marked as doing their transport work, also while the send completes, so nothing + * logged about the upload is gathered, and once it is sent the logger may queue its next batch. + * + * @param config configuration of the running SDK + * @param request the oldest queued request + * @param uploadsGatheredLogs whether the request carries a batch of gathered log lines + * @return whether the request was handed over for sending or removed + */ + private boolean prepareAndSend(final InternalConfig config, final Request request, final boolean uploadsGatheredLogs) { + L.d("[Networking] Preparing request: " + request); + final Boolean check = SDKCore.instance.isRequestReady(request); + if (check == null) { + L.d("[Networking] Request is not ready yet: " + request); + return false; + } else if (check.equals(Boolean.FALSE)) { + L.d("[Networking] Request won't be ready, removing: " + request); + Storage.remove(config, request); + return true; + } + + if (request.params.has("rr")) { + request.params.remove("rr"); + } + request.params.add("rr", storageForRequestQueue.remaningRequests()); + final AtomicLong responseTimeMs = new AtomicLong(-1L); + tasks.run(transport.send(request, responseTimeMs::set), result -> { + L.setOwnTransportWork(uploadsGatheredLogs); + try { + L.d("[Networking] Request " + request.storageId() + " sent?: " + result + ", response time [" + responseTimeMs.get() + "] ms"); + if (result) { + storageForRequestQueue.removeRequest(request); + if (uploadsGatheredLogs) { + L.onLogBatchSent(); + } + if (!backOffIfTheServerIsSlow(config, request, responseTimeMs.get())) { + check(config); + } + } + } finally { + L.setOwnTransportWork(false); + } + }); + return true; + } + + /** + * Reads the oldest queued request, first dropping, unsent, every queued request older than the + * request drop age of the SDK behavior settings. + * + * @param config configuration of the running SDK + * @param followsOwnTransportWork whether the log gathering's own transport work started the drain + * @return the request to send next, {@code null} when the queue is empty or a request could not be dropped + */ + @Nullable + private Request nextRequestWithinDropAge(@Nonnull InternalConfig config, boolean followsOwnTransportWork) { + Request request = readNextRequest(followsOwnTransportWork); + while (request != null && isOlderThanDropAge(config, request)) { + if (!dropWithoutSending(config, request)) { + return null; + } + request = readNextRequest(followsOwnTransportWork); + } + return request; + } + + /** + * Reads the oldest queued request. While the log gathering's own transport work started the + * drain, the thread is marked during the read, so the read it logs is not gathered, while a + * request dropped meanwhile is still reported. + * + * @param followsOwnTransportWork whether the log gathering's own transport work started the drain + * @return the request, {@code null} when the queue is empty + */ + @Nullable + private Request readNextRequest(boolean followsOwnTransportWork) { + L.setOwnTransportWork(followsOwnTransportWork); + try { + return storageForRequestQueue.getNextRequest(); + } finally { + L.setOwnTransportWork(false); + } + } + + /** + * Whether a queued request is older than the request drop age of the SDK behavior settings, a + * drop age of {@code 0} keeping every request. Never in backend mode, where the settings are + * inert and a request carries the time the application recorded it at. + * + * @param config configuration of the running SDK + * @param request the queued request + * @return {@code true} when the request has to be dropped without being sent + */ + private boolean isOlderThanDropAge(@Nonnull InternalConfig config, @Nonnull Request request) { + if (backendMode) { + return false; + } + + int dropAgeHours = config.getConfigurationProvider().getRequestDropAgeHours(); + return dropAgeHours > 0 && requestAgeMs(request) > dropAgeHours * MS_IN_HOUR; + } + + /** + * Removes a queued request without sending it, then tells the module that owns it, with a + * {@code null} response and {@link Transport#NO_RESPONSE_CODE}, so nothing keeps waiting for + * its response. + * + * @param config configuration of the running SDK + * @param request the request to drop + * @return whether the request was removed + */ + private boolean dropWithoutSending(@Nonnull InternalConfig config, @Nonnull Request request) { + L.w("[Networking] dropWithoutSending, request [" + request.storageId() + "] is older than the request drop age of [" + config.getConfigurationProvider().getRequestDropAgeHours() + "] hours set by the SDK behavior settings, dropping it without sending"); + if (!Boolean.TRUE.equals(storageForRequestQueue.removeRequest(request))) { + L.e("[Networking] dropWithoutSending, failed to remove request [" + request.storageId() + "], the queue drain stops here"); + return false; + } + if (request.params.has(Log.keySdkLogs)) { + L.onLogBatchSent(); + } + + Class requestOwner = request.owner(); + SDKCore core = SDKCore.instance; + if (requestOwner != null && core != null) { + try { + core.onRequestCompleted(request, null, Transport.NO_RESPONSE_CODE, requestOwner); + } catch (Exception e) { + L.e("[Networking] dropWithoutSending, failed to tell the owner of request [" + request.storageId() + "] that it was dropped, [" + e + "]"); + } + } + return true; + } + + /** + * How long ago a request was recorded, read from its {@code timestamp} parameter, or from its + * storage ID, which is taken from the clock when the request is created, when it has none. + * + * @param request the request + * @return the age in milliseconds, negative for a request dated in the future + */ + private long requestAgeMs(@Nonnull Request request) { + Long recordedAtMs = request.storageId(); + String timestamp = request.params == null ? null : request.params.get(TIMESTAMP_PARAM); + if (timestamp != null) { + try { + recordedAtMs = Long.parseLong(timestamp); + } catch (NumberFormatException e) { + L.w("[Networking] requestAgeMs, request [" + request.storageId() + "] has a timestamp that is not a number [" + timestamp + "], dating it by its storage ID"); + } + } + + long nowMs = System.currentTimeMillis(); + return recordedAtMs == null ? 0L : nowMs - recordedAtMs; + } + + /** + * Holds the request queue back for the backoff duration of the SDK behavior settings when the + * backoff mechanism is on and the server answered slowly: the response took at least the + * accepted timeout, the request is not older than the backoff request age, and the requests + * still queued are at most the given share of the request queue size. Never in backend mode, + * where the settings are inert. + * + * @param config configuration of the running SDK + * @param request the request the server just accepted, already removed from the queue + * @param responseTimeMs how long the server took to answer it, in milliseconds + * @return whether the queue is now held back + */ + private boolean backOffIfTheServerIsSlow(@Nonnull InternalConfig config, @Nonnull Request request, long responseTimeMs) { + if (backendMode) { + return false; + } + + ConfigurationProvider provider = config.getConfigurationProvider(); + if (!provider.getBOMEnabled() || responseTimeMs < provider.getBOMAcceptedTimeoutSeconds() * 1000L) { + return false; + } + + if (requestAgeMs(request) > provider.getBOMRequestAge() * MS_IN_HOUR) { + L.v("[Networking] backOffIfTheServerIsSlow, the server took [" + responseTimeMs + "] ms to answer request [" + request.storageId() + "], but the request is older than [" + provider.getBOMRequestAge() + "] hours, not backing off"); + return false; + } + + int queuedRequests = queuedRequestCount(); + if (queuedRequests < 0) { + return false; + } + + double queueShare = provider.getRequestQueueMaxSize() * provider.getBOMRQPercentage(); + if (queuedRequests > queueShare) { + L.v("[Networking] backOffIfTheServerIsSlow, the server took [" + responseTimeMs + "] ms to answer request [" + request.storageId() + "], but [" + queuedRequests + "] requests remain queued, more than [" + queueShare + "], not backing off"); + return false; + } + + int durationSeconds = provider.getBOMDuration(); + L.i("[Networking] backOffIfTheServerIsSlow, the server took [" + responseTimeMs + "] ms to answer request [" + request.storageId() + "] and [" + queuedRequests + "] requests remain queued, holding the request queue back for [" + durationSeconds + "] seconds"); + return startBackoff(config, durationSeconds); + } + + /** + * How many requests the queue holds. + * + * @return the number of queued requests, {@code -1} when they could not be counted + */ + private int queuedRequestCount() { + try { + Integer remaining = storageForRequestQueue.remaningRequests(); + return remaining == null ? -1 : remaining + 1; + } catch (RuntimeException e) { + L.w("[Networking] queuedRequestCount, failed to count the queued requests, not backing off, [" + e + "]"); + return -1; + } + } + + /** + * Holds the request queue back for the given time, after which it drains again. + * + * @param config configuration of the running SDK + * @param durationSeconds how long to hold the queue back + * @return whether the queue is held back, {@code false} when the SDK is stopping + */ + private boolean startBackoff(@Nonnull final InternalConfig config, int durationSeconds) { + backedOff = true; + try { + backoffScheduler.schedule(() -> endBackoff(config), durationSeconds, TimeUnit.SECONDS); + return true; + } catch (RejectedExecutionException e) { + backedOff = false; + L.d("[Networking] startBackoff, the SDK is stopping, the request queue is not held back"); + return false; + } + } + + /** + * Ends a backoff and lets the request queue drain again, unless the SDK stopped meanwhile. + * + * @param config configuration of the running SDK + */ + private void endBackoff(@Nonnull InternalConfig config) { + backedOff = false; + if (shutdown) { + return; + } + + L.d("[Networking] endBackoff, the backoff is over, the request queue drains again"); + try { + check(config); + } catch (RuntimeException e) { + L.w("[Networking] endBackoff, failed to restart the request queue drain, [" + e + "]"); + } + } + + /** + * Stops sending. A backoff that is still running never ends in a drain, and a request that is + * being sent may finish. + * + * @param config configuration of the SDK being stopped + */ @Override public void stop(InternalConfig config) { shutdown = true; + if (backoffScheduler != null) { + backoffScheduler.shutdownNow(); + try { + if (!backoffScheduler.awaitTermination(1, TimeUnit.SECONDS)) { + L.w("[Networking] stop, the backoff scheduler did not stop in time"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } tasks.shutdown(); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java index d446315d..8d633c1d 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.Nonnull; import ly.count.sdk.java.Config; /** @@ -32,6 +33,11 @@ public class InternalConfig extends Config { protected IdGenerator viewIdGenerator; protected IdGenerator eventIdGenerator; protected ViewIdProvider viewIdProvider; + /** + * The SDK behavior settings of the running SDK, registered by {@link ModuleConfiguration} when it + * initializes. Read it through {@link #getConfigurationProvider()}. + */ + volatile ConfigurationProvider configProvider; /** * Shouldn't be used! @@ -144,13 +150,47 @@ public void setDefaultNetworking(boolean defaultNetworking) { } /** - * This feature is not yet implemented - * and always return true + * The SDK behavior settings in effect. While no {@link ModuleConfiguration} is registered on + * this configuration, as for a module built on its own in a test, the settings come from this + * configuration alone. + * + * @return the settings, never {@code null} + */ + @Nonnull + ConfigurationProvider getConfigurationProvider() { + ConfigurationProvider provider = configProvider; + if (provider != null) { + return provider; + } + return ModuleConfiguration.developerDefaults(this); + } + + /** + * Whether the SDK is currently allowed to perform network requests, the {@code networking} + * switch of the SDK behavior settings. * - * @return true + * @return {@code true} if networking is allowed */ public boolean getNetworkingEnabled() { - return true; + return getConfigurationProvider().getNetworkingEnabled(); + } + + public String getSdkBehaviorSettings() { + return sdkBehaviorSettings; + } + + public boolean isSdkBehaviorSettingsRequestsDisabled() { + return sdkBehaviorSettingsRequestsDisabled; + } + + /** + * Whether the developer left the backoff mechanism on, the value the {@code bom} setting of the + * SDK behavior settings starts from. + * + * @return {@code false} after {@link Config#disableBackoffMechanism()} + */ + boolean isBackoffMechanismEnabled() { + return backoffMechanismEnabled; } /** diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/JsonFileStorage.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/JsonFileStorage.java index 9fb214ea..51d17cc5 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/JsonFileStorage.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/JsonFileStorage.java @@ -3,8 +3,12 @@ import java.io.BufferedWriter; import java.io.File; import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import javax.annotation.Nonnull; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; public class JsonFileStorage { @@ -21,9 +25,22 @@ public class JsonFileStorage { public JsonFileStorage(@Nonnull final File file, @Nonnull Log logger) { this.logger = logger; this.file = file; + deleteLeftoverTemporaryFile(); this.json = readJsonFile(file); } + /** + * Deletes the temporary file of a save that was stopped before its move. The json file still + * holds the previous save then, and a leftover file would be taken for other stored SDK data. + */ + private void deleteLeftoverTemporaryFile() { + try { + Files.deleteIfExists(temporaryFileOf(file).toPath()); + } catch (IOException e) { + logger.w("[JsonFileStorage] deleteLeftoverTemporaryFile, Failed to delete the temporary file, reason: [" + e + "]"); + } + } + /** * If key exists changes its value, otherwise adds new key-value pair * Null key or value is not allowed @@ -32,24 +49,62 @@ public JsonFileStorage(@Nonnull final File file, @Nonnull Log logger) { * @param key to set * @param value to add */ - public void add(@Nonnull final String key, @Nonnull Object value) { + public synchronized void add(@Nonnull final String key, @Nonnull Object value) { logger.i("[JsonFileStorage] add, Adding key: [" + key + "], value: [" + value + "]"); json.put(key, value); } /** - * Saves changes to the disk/db/memory + * Saves changes to the disk/db/memory. The data is written next to the file and then moved over + * it, so a process stopped while saving leaves the previous file whole. When the data cannot be + * serialized, the file keeps what it had. */ - public void save() { + public synchronized void save() { logger.i("[JsonFileStorage] save, Saving json file: [" + file.getAbsolutePath() + "]"); - try (BufferedWriter writer = Files.newBufferedWriter(file.toPath())) { - writer.write(json.toString()); + // toString returns null instead of throwing, so a failed serialization has to be caught before anything is written + String content = json.toString(); + if (content == null) { + logger.e("[JsonFileStorage] save, Failed to serialize the data, the json file is left as it is: [" + file.getAbsolutePath() + "]"); + return; + } + + File temporary = temporaryFileOf(file); + try { + try (BufferedWriter writer = Files.newBufferedWriter(temporary.toPath())) { + writer.write(content); + } + replaceFile(temporary, file); } catch (IOException e) { logger.e("[JsonFileStorage] save, Failed to save json file, reason: [" + e.getMessage() + "]"); } } + /** + * The file a save writes before moving it over the json file. + * + * @param file the json file + * @return the temporary file next to it + */ + static File temporaryFileOf(@Nonnull File file) { + return new File(file.getAbsolutePath() + ".tmp"); + } + + /** + * Moves a file over another, atomically where the file system supports it. + * + * @param source the file to move + * @param target the file it replaces + * @throws IOException when the move fails + */ + private static void replaceFile(@Nonnull File source, @Nonnull File target) throws IOException { + try { + Files.move(source.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + Files.move(source.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } + /** * Removes key-value pair * Null key is not allowed @@ -57,7 +112,7 @@ public void save() { * * @param key to remove */ - public void delete(@Nonnull final String key) { + public synchronized void delete(@Nonnull final String key) { if (!json.has(key)) { logger.v("[JsonFileStorage] delete, Nothing to delete"); } @@ -71,7 +126,7 @@ public void delete(@Nonnull final String key) { * @param key to set * @param value to add */ - public void addAndSave(@Nonnull final String key, @Nonnull Object value) { + public synchronized void addAndSave(@Nonnull final String key, @Nonnull Object value) { add(key, value); save(); } @@ -82,7 +137,7 @@ public void addAndSave(@Nonnull final String key, @Nonnull Object value) { * * @param key to remove */ - public void deleteAndSave(@Nonnull final String key) { + public synchronized void deleteAndSave(@Nonnull final String key) { delete(key); save(); } @@ -94,19 +149,53 @@ public void deleteAndSave(@Nonnull final String key) { * @param key to get * @return value */ - public Object get(@Nonnull final String key) { + public synchronized Object get(@Nonnull final String key) { return json.opt(key); } /** - * Returns JSONObject value for the key + * Returns a copy of the JSONObject value for the key. Changing the copy changes nothing stored: + * hand it back through {@link #add(String, Object)} to store it, so another thread that saves + * never serializes an object while it is being changed. * * @param key to get * @param defaultValue to return if key not found - * @return value, if key not found returns defaultValue + * @return a deep copy of the value, if key not found returns defaultValue + */ + public synchronized JSONObject getJsonObj(@Nonnull final String key, final JSONObject defaultValue) { + JSONObject value = json.optJSONObject(key, null); + if (value == null) { + return defaultValue; + } + return (JSONObject) deepCopy(value); + } + + /** + * Copies a JSON value, every nested object and array included, keeping every other value as it is. + * + * @param value the value to copy + * @return the copy, or the value itself when it holds nothing that can change */ - public JSONObject getJsonObj(@Nonnull final String key, final JSONObject defaultValue) { - return json.optJSONObject(key, defaultValue); + private static Object deepCopy(final Object value) { + if (value instanceof JSONObject) { + JSONObject source = (JSONObject) value; + JSONObject copy = new JSONObject(); + for (String key : source.keySet()) { + copy.put(key, deepCopy(source.opt(key))); + } + return copy; + } + + if (value instanceof JSONArray) { + JSONArray source = (JSONArray) value; + JSONArray copy = new JSONArray(); + for (int i = 0; i < source.length(); i++) { + copy.put(deepCopy(source.opt(i))); + } + return copy; + } + + return value; } /** @@ -115,7 +204,7 @@ public JSONObject getJsonObj(@Nonnull final String key, final JSONObject default * @param key to get * @return value, if key not found returns null */ - public String getString(@Nonnull final String key) { + public synchronized String getString(@Nonnull final String key) { return getString(key, null); } @@ -126,7 +215,7 @@ public String getString(@Nonnull final String key) { * @param defaultValue to return if key not found * @return value, if key not found returns defaultValue */ - public String getString(@Nonnull final String key, final String defaultValue) { + public synchronized String getString(@Nonnull final String key, final String defaultValue) { return json.optString(key, defaultValue); } @@ -137,21 +226,21 @@ public String getString(@Nonnull final String key, final String defaultValue) { * @param defaultValue to return if key not found * @return value, if key not found returns defaultValue */ - public int getInt(@Nonnull final String key, final int defaultValue) { + public synchronized int getInt(@Nonnull final String key, final int defaultValue) { return json.optInt(key, defaultValue); } /** * Clears all data */ - public void clear() { + public synchronized void clear() { json.clear(); } /** * Clears all data and saves changes to the disk/db/memory */ - public void clearAndSave() { + public synchronized void clearAndSave() { clear(); save(); } @@ -161,7 +250,7 @@ public void clearAndSave() { * * @return number of key-value pairs */ - public int size() { + public synchronized int size() { return json.length(); } @@ -182,6 +271,9 @@ private JSONObject readJsonFile(@Nonnull final File file) { } catch (IOException e) { logger.e("[JsonFileStorage] readJsonFile, Failed to read json file, reason: [" + e.getMessage() + "]"); return new JSONObject(); + } catch (JSONException e) { + logger.e("[JsonFileStorage] readJsonFile, The json file is not valid json, starting with empty storage that replaces it on the next save: [" + file.getAbsolutePath() + "], reason: [" + e.getMessage() + "]"); + return new JSONObject(); } } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/Log.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/Log.java index 2fd4a8f7..aee71824 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/Log.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/Log.java @@ -1,24 +1,184 @@ package ly.count.sdk.java.internal; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.IntSupplier; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import ly.count.sdk.java.Config; +import ly.count.sdk.java.internal.ConfigurationProvider.LogGatheringState; +import org.json.JSONObject; /** * Logging module. Exposes static functions for simplicity, thus can be used only from some point * in time when {@link Config} is created and {@link ModuleBase}s are up. + *

+ * The logger also gathers the SDK's own lines for the server when the {@code lg} directive of the + * SDK behavior settings asks for them, and uploads them as the {@code sdk_logs} parameter of + * requests of their own, shaped {@code {"i":gatherId,"d":dropped,"l":[{"t":ms,"l":"e","m":message}]}}. + * Every level is captured, whatever is printed. Capture starts with the first line, into a + * provisional buffer, since the init lines are the ones worth having: a live directive adopts the + * buffer filtered to its levels, or drops it. Nothing of it is ever stored. A line whose consent + * is not given when it is logged waits apart until it is, see {@link #logLineConsentFeatures(String)}. */ public class Log { + static final String keySdkLogs = "sdk_logs"; + //a line that carries an uploaded batch is never gathered, or each batch would nest the previous one + static final String transportMarker = keySdkLogs + "="; + + static final String keyBatchGatherId = "i"; + static final String keyBatchDroppedCount = "d"; + static final String keyBatchLines = "l"; + static final String keyLineTimestamp = "t"; + static final String keyLineLevel = "l"; + static final String keyLineMessage = "m"; + + //the server keeps at most this many characters of a message + static final int maxMessageLength = 4096; + //a ceiling on the characters held, since the line ceiling alone does not bound the memory + static final int gatheredCharCeiling = 128 * 1024; + static final String DELIVERY_THREAD_NAME = "Countly-LogDelivery"; + + //the longest bracketed name a line starts with, so finding it never scans a long line to its end + private static final int maxLogLineNameLength = 64; + //the consent a line of no feature needs, as it can quote any feature's data, such as a whole request + static final int UNATTRIBUTED_LINE_CONSENT = CoreFeature.Events.getIndex() | CoreFeature.UserProfiles.getIndex(); + + /** + * The consent a gathered line needs while consent is required, by the name its message starts + * with in brackets, as in {@code "[ModuleEvents] ..."}: the features any one of whose consent + * lets the line go. A line of any other name needs {@link #UNATTRIBUTED_LINE_CONSENT}. + */ + private static final Map logLineConsent; + + static { + Map consent = new HashMap<>(); + mapLogLineConsent(consent, CoreFeature.Events, "ModuleEvents", "Events", "EventImpl", "EventQueue", "TimedEvents"); + mapLogLineConsent(consent, CoreFeature.Sessions, "ModuleSessions", "SessionImpl"); + mapLogLineConsent(consent, CoreFeature.Views, "ModuleViews", "Views", "ViewImpl", "View"); + mapLogLineConsent(consent, CoreFeature.CrashReporting, "ModuleCrash", "Crashes", "CrashImpl"); + mapLogLineConsent(consent, CoreFeature.Location, "ModuleLocation", "Location"); + mapLogLineConsent(consent, CoreFeature.UserProfiles, "ModuleUserProfile", "UserProfile", "UserEditorImpl"); + mapLogLineConsent(consent, CoreFeature.RemoteConfig, "ModuleRemoteConfig", "RemoteConfig", "RemoteConfigValueStore"); + mapLogLineConsent(consent, CoreFeature.Feedback, "ModuleFeedback", "Feedback"); + mapLogLineConsent(consent, CoreFeature.Content, "ModuleContent", "Content", "ContentParser", "ContentRequestBuilder", "WidgetActionParser"); + logLineConsent = Collections.unmodifiableMap(consent); + } + private final LogCallback logListener; private final Config.LoggingLevel loggingLevel; + //the lowest level printed: the developer level unless the SDK behavior settings override it + private volatile Config.LoggingLevel printLevel; + + //guards the buffer and the directive mirror; nothing is logged and nothing is called out while it is held + private final Object logBufferLock = new Object(); + //held while batches are taken and queued, so ending the gathering waits for a delivery in flight + private final Object logDeliveryLock = new Object(); + private final Object logDeliveryExecutorLock = new Object(); + + private final ArrayDeque gatheredLines = new ArrayDeque<>(); + int droppedLogLineCount = 0; + int gatheredChars = 0; + //lines whose consent was not given when they were logged, kept out of every batch until it is + private final ArrayDeque linesAwaitingConsent = new ArrayDeque<>(); + int awaitingConsentChars = 0; + //orders the lines of both lists, as the millisecond timestamps of a burst do not + private long nextLineSequence = 0; + //the last batch of a gather the server stopped, taken under the lock and queued outside it + private LogBatch pendingTailBatch = null; + + //the directive mirror, so capturing a line never reads the settings; the state alone is read without the lock + volatile LogGatheringState logGatheringState = LogGatheringState.UNDECIDED; + String logGatherId = null; + String logGatherLevels = ModuleConfiguration.logGatheringAllLevels; + int logGatherBatchSize = ModuleConfiguration.logGatheringDefaultBatchSize; + private volatile boolean logGatheringEnded = false; + + private final ThreadLocal ownTransportWork = new ThreadLocal<>(); + private final AtomicBoolean logDeliveryPending = new AtomicBoolean(false); + private final AtomicBoolean partialLogDeliveryRequested = new AtomicBoolean(false); + //the storage IDs of the batches this logger queued, until the request queue no longer holds them + private final Set queuedLogBatchIds = ConcurrentHashMap.newKeySet(); + //a delivery left lines for the batch waiting in the request queue, so the send of that batch resumes it + private final AtomicBoolean logDeliveryAwaitsSentBatch = new AtomicBoolean(false); + volatile ExecutorService logDeliveryExecutor = null; + private volatile boolean sdkInitFinished = false; + + private volatile ConfigurationProvider configProvider = null; + private volatile IntSupplier givenConsents = null; + private volatile LogBatchQueue logBatchQueue = null; + + /** + * The request queue as the log gathering uses it: one request per batch of gathered log lines. + */ + interface LogBatchQueue { + /** + * Queues the batch. + * + * @param payload the {@code sdk_logs} value + * @return the storage ID of the request carrying the batch, {@code null} when the queue refused it + */ + @Nullable + Long queueLogBatch(@Nonnull String payload); + + /** + * Whether a request still waits in the queue, until it is sent or dropped. + * + * @param requestId the storage ID {@link #queueLogBatch(String)} returned + * @return {@code true} while the request waits + */ + boolean isQueued(@Nonnull Long requestId); + } public Log(Config.LoggingLevel loggingLevel, LogCallback logListener) { if (loggingLevel == null) { throw new NullPointerException("Logging level can't null."); } this.loggingLevel = loggingLevel; + this.printLevel = loggingLevel; this.logListener = logListener; } + /** + * Applies the {@code log} switch of the SDK behavior settings to what is printed. Switched off, + * nothing is printed. Switched on, the developer level is printed, or every level when the + * developer level is {@link Config.LoggingLevel#OFF}. The log listener receives every line either way. + * + * @param loggingEnabled the resolved {@code log} switch + */ + void setLoggingEnabled(boolean loggingEnabled) { + if (!loggingEnabled) { + printLevel = Config.LoggingLevel.OFF; + } else if (loggingLevel == Config.LoggingLevel.OFF) { + printLevel = Config.LoggingLevel.VERBOSE; + } else { + printLevel = loggingLevel; + } + } + + /** + * The lowest level printed, which the SDK behavior settings can change. + * + * @return the level, {@link Config.LoggingLevel#OFF} while nothing is printed + */ + Config.LoggingLevel getPrintLevel() { + return printLevel; + } + /** * {@link Config.LoggingLevel} level logging * @@ -27,6 +187,7 @@ public Log(Config.LoggingLevel loggingLevel, LogCallback logListener) { public void d(String logMessage) { print("[DEBUG] [Countly]\t" + logMessage, Config.LoggingLevel.DEBUG); informListener(logMessage, Config.LoggingLevel.DEBUG); + captureLogLine(logMessage, 'd'); } /** @@ -37,6 +198,7 @@ public void d(String logMessage) { public void i(String logMessage) { print("[INFO] [Countly]\t" + logMessage, Config.LoggingLevel.INFO); informListener(logMessage, Config.LoggingLevel.INFO); + captureLogLine(logMessage, 'i'); } /** @@ -47,6 +209,7 @@ public void i(String logMessage) { public void w(String logMessage) { print("[WARN] [Countly]\t" + logMessage, Config.LoggingLevel.WARN); informListener(logMessage, Config.LoggingLevel.WARN); + captureLogLine(logMessage, 'w'); } /** @@ -57,6 +220,7 @@ public void w(String logMessage) { public void e(String logMessage) { print("[ERROR] [Countly]\t" + logMessage, Config.LoggingLevel.ERROR); informListener(logMessage, Config.LoggingLevel.ERROR); + captureLogLine(logMessage, 'e'); } /** @@ -67,10 +231,11 @@ public void e(String logMessage) { public void v(String logMessage) { print("[VERBOSE] [Countly]\t" + logMessage, Config.LoggingLevel.VERBOSE); informListener(logMessage, Config.LoggingLevel.VERBOSE); + captureLogLine(logMessage, 'v'); } private void print(String msg, Config.LoggingLevel level) { - if (level != null && loggingLevel.prints(level)) { + if (level != null && printLevel.prints(level)) { System.out.println(msg); } } @@ -80,4 +245,975 @@ private void informListener(String msg, Config.LoggingLevel level) { logListener.LogHappened(msg, level); } } + + /** + * Hands the logger what it needs to act on the log gathering directive, lets the lines logged + * so far whose consent is given join the gather, then applies the directive in effect. Setters + * rather than constructor parameters, as the logger exists before any of them. + * + * @param configProvider the SDK behavior settings, which hold the directive + * @param givenConsents the {@link CoreFeature} indexes whose consent is given, every bit while + * no consent is required; read inside log calls, so it must never log + * @param logBatchQueue queues one batch as a request of its own + */ + void setLogGatheringProviders(@Nonnull ConfigurationProvider configProvider, @Nonnull IntSupplier givenConsents, @Nonnull LogBatchQueue logBatchQueue) { + v("[Log] setLogGatheringProviders, held lines:[" + heldLogLineCount() + "], awaiting consent:[" + awaitingConsentLineCount() + "]"); + this.givenConsents = givenConsents; + this.logBatchQueue = logBatchQueue; + this.configProvider = configProvider; + releaseConsentedLogLines(); + applyLogGatheringDirective("init"); + } + + /** + * Applies the directive the settings hold to what is held. An armed directive adopts the held + * lines filtered to its levels, and a new gather id drops the lines of the previous gather. A + * decision against gathering stops the capture: the lines a running gather held are queued once, + * with its id, and the lines held while undecided are dropped. Called after every decision, and + * ignored once the gathering ended. + * + * @param source what decided, for the log line only + */ + void applyLogGatheringDirective(@Nonnull String source) { + if (logGatheringEnded) { + return; + } + + ConfigurationProvider provider = configProvider; + if (provider == null) { + v("[Log] applyLogGatheringDirective, no configuration provider yet, source:[" + source + "]"); + return; + } + + LogGatheringState state = provider.getLogGatheringState(); + String gatherId = provider.getLogGatheringId(); + String levels = provider.getLogGatheringLevels(); + int batchSize = provider.getLogGatheringBatchSize(); + + if (!adoptLogGatheringDirective(state, gatherId, levels, batchSize)) { + return; + } + + d("[Log] applyLogGatheringDirective, source:[" + source + "], state:[" + state + "], id:[" + gatherId + "], levels:[" + levels + "], batch size:[" + batchSize + "], held lines:[" + heldLogLineCount() + "]"); + + if (state != LogGatheringState.GATHERING) { + shutdownLogDeliveryAfterTail(); + return; + } + + ensureLogDeliveryExecutor(); + scheduleLogDelivery(false); + } + + /** + * Lets the lines held during init go out, as the request queue is usable from here on. + */ + void onSdkInitFinished() { + sdkInitFinished = true; + if (isGatheringLogs()) { + d("[Log] onSdkInitFinished, uploading the lines gathered during init"); + scheduleLogDelivery(true); + } + } + + /** + * Uploads what is held, on a timer tick, so a buffer that never fills a batch still goes out. + */ + void flushGatheredLogs() { + if (isGatheringLogs() && heldLogLineCount() > 0) { + scheduleLogDelivery(true); + } + } + + /** + * Tells the logger that a request carrying a log batch left the request queue, sent or dropped + * unsent. One batch waits in the queue at a time, so while nothing is sent the lines stay in the + * capped buffer instead of filling the queue; a delivery that left lines behind for the waiting + * batch resumes now, a partial batch included. Called by the networking, on a thread marked as + * doing this feature's own transport work, and never logs. + */ + void onLogBatchSent() { + if (logDeliveryAwaitsSentBatch.compareAndSet(true, false) && isGatheringLogs()) { + scheduleLogDelivery(true); + } + } + + /** + * Stops the gather before the device ID changes without merge: queues every held line on the + * calling thread, past the one waiting batch, so each request carries the current device ID, + * then drops the lines still waiting for their consent and stops the capture, so nothing the + * next user does is gathered for the current gather. A later live directive can arm it again. + */ + void stopLogGatheringForDeviceIdChange() { + queueHeldLinesAndStop("stopLogGatheringForDeviceIdChange", true, false); + d("[Log] stopLogGatheringForDeviceIdChange, log gathering stopped before the device ID change"); + } + + /** + * Drops the held lines whose consent is no longer given, as after a consent was removed, the + * last lines of a stopped gather included, so nothing gathered for a feature goes out once its + * consent is withdrawn. They are counted as dropped. + */ + void dropLinesWithoutConsent() { + IntSupplier consents = givenConsents; + if (consents == null) { + return; + } + int given = consents.getAsInt(); + + synchronized (logBufferLock) { + Iterator iterator = gatheredLines.iterator(); + while (iterator.hasNext()) { + LogLine line = iterator.next(); + if (!isConsentGiven(line.consentFeatures, given)) { + iterator.remove(); + gatheredChars -= line.message.length(); + droppedLogLineCount++; + } + } + + LogBatch tail = pendingTailBatch; + if (tail != null) { + List kept = new ArrayList<>(tail.lines.size()); + for (LogLine line : tail.lines) { + if (isConsentGiven(line.consentFeatures, given)) { + kept.add(line); + } + } + pendingTailBatch = kept.isEmpty() ? null : new LogBatch(tail.gatherId, tail.dropped + tail.lines.size() - kept.size(), kept); + } + } + } + + /** + * Moves the lines whose consent is given by now out of the waiting list into the buffer, in + * capture order with the lines already there, and asks for a delivery when a full batch is + * held. Called when the providers are set and whenever consent is given. + */ + void releaseConsentedLogLines() { + IntSupplier consents = givenConsents; + if (consents == null) { + return; + } + int given = consents.getAsInt(); + + boolean fullBatch; + synchronized (logBufferLock) { + List released = new ArrayList<>(); + Iterator iterator = linesAwaitingConsent.iterator(); + while (iterator.hasNext()) { + LogLine line = iterator.next(); + if (isConsentGiven(line.consentFeatures, given)) { + iterator.remove(); + awaitingConsentChars -= line.message.length(); + released.add(line); + } + } + if (!released.isEmpty()) { + mergeIntoBufferLocked(released); + } + fullBatch = isGatheringLogs() && gatheredLines.size() >= logGatherBatchSize; + } + + if (fullBatch) { + scheduleLogDelivery(false); + } + } + + /** + * Marks the calling thread as doing this feature's own transport work, so nothing it logs + * meanwhile is gathered, or every upload would report on itself. + * + * @param transporting {@code true} to mark the thread, {@code false} to clear the mark + */ + void setOwnTransportWork(boolean transporting) { + if (transporting) { + ownTransportWork.set(Boolean.TRUE); + } else { + ownTransportWork.remove(); + } + } + + /** + * Whether the calling thread is marked as doing this feature's own transport work. + * + * @return {@code true} while the thread is marked + */ + boolean isOwnTransportWork() { + return Boolean.TRUE.equals(ownTransportWork.get()); + } + + /** + * Ends log gathering for good, as the SDK that owns this logger stops. A stop that keeps the SDK + * data queues what is held first, as far as tracking and consent allow, since nothing held + * outlives the SDK; a stop that clears the data discards it. Waits for a delivery in flight, so + * no batch of this logger is queued afterwards, and releases the delivery thread. Lines logged + * afterwards, by threads that outlive the SDK, are neither captured nor queued. + * + * @param queueHeldLines {@code true} to queue the held lines first, {@code false} to discard them + */ + void endLogGathering(boolean queueHeldLines) { + queueHeldLinesAndStop("endLogGathering", queueHeldLines, true); + configProvider = null; + givenConsents = null; + logBatchQueue = null; + v("[Log] endLogGathering, log gathering ended"); + } + + /** + * Queues what is held on the calling thread when asked, the held lines and then the last batch + * of a gather the server stopped, so each request carries the current device ID, then stops the + * capture and drops what is left, lines waiting for their consent included. Waits for a + * delivery in flight first, and releases the delivery thread in the same step as the stop, so a + * directive applied meanwhile either runs before it or starts a delivery thread of its own. + * + * @param caller the caller, for the log lines + * @param queueHeldLines {@code true} to queue what is held first, {@code false} to drop it + * @param endForGood {@code true} when the SDK stops, so nothing is gathered again, {@code false} + * when a later live directive may arm a gather again + */ + private void queueHeldLinesAndStop(@Nonnull String caller, boolean queueHeldLines, boolean endForGood) { + ExecutorService executor = null; + Boolean previousMark = ownTransportWork.get(); + setOwnTransportWork(true); + try { + synchronized (logDeliveryLock) { + try { + LogBatchQueue queue = logBatchQueue; + if (queueHeldLines && queue != null) { + if (isGatheringLogs()) { + int heldLines = heldLogLineCount(); + d("[Log] " + caller + ", queueing the [" + heldLines + "] held lines"); + deliverHeldLogBatches(heldLines, false); + } + LogBatch tail = takePendingTailBatch(); + if (tail != null) { + queueTail(queue, tail); + } + } + } catch (RuntimeException e) { + e("[Log] " + caller + ", failed to queue the held lines, [" + e + "]"); + } finally { + synchronized (logBufferLock) { + if (!logGatheringEnded) { + logGatheringEnded = endForGood; + logGatheringState = LogGatheringState.NOT_GATHERING; + logGatherId = null; + logGatherLevels = ModuleConfiguration.logGatheringAllLevels; + logGatherBatchSize = ModuleConfiguration.logGatheringDefaultBatchSize; + pendingTailBatch = null; + clearBufferLocked(); + executor = takeLogDeliveryExecutor(); + } + } + } + } + } finally { + setOwnTransportWork(Boolean.TRUE.equals(previousMark)); + } + + if (executor != null) { + v("[Log] " + caller + ", stopping the log batch delivery executor"); + executor.shutdown(); + } + } + + /** + * Whether lines are still held for the server, which is true until something decides against + * gathering. + * + * @return {@code true} while lines are captured + */ + boolean isCapturingLogs() { + return logGatheringState != LogGatheringState.NOT_GATHERING; + } + + /** + * Whether a live directive armed the gathering. + * + * @return {@code true} while gathering + */ + boolean isGatheringLogs() { + return logGatheringState == LogGatheringState.GATHERING; + } + + /** + * How many lines are held. + * + * @return the number of lines in the buffer + */ + int heldLogLineCount() { + synchronized (logBufferLock) { + return gatheredLines.size(); + } + } + + /** + * How many lines wait for their consent, outside the buffer. + * + * @return the number of waiting lines + */ + int awaitingConsentLineCount() { + synchronized (logBufferLock) { + return linesAwaitingConsent.size(); + } + } + + /** + * Holds a line for the server: every level while nothing is decided, the directive's levels + * while gathering, nothing while not gathering. Lines that carry an uploaded batch and lines of a + * thread doing this feature's own transport work are skipped. A line whose consent is not given + * waits apart until it is. A full batch is handed to the delivery thread. + * + * @param message the line as logged + * @param level the level character: e, w, i, d or v + */ + private void captureLogLine(@Nullable String message, char level) { + if (logGatheringState == LogGatheringState.NOT_GATHERING) { + return; + } + if (Boolean.TRUE.equals(ownTransportWork.get())) { + return; + } + String msg = message == null ? "" : message; + if (msg.contains(transportMarker)) { + return; + } + + int consentFeatures = logLineConsentFeatures(msg); + boolean consentGiven = isConsentGiven(consentFeatures); + boolean batchIsFull = false; + synchronized (logBufferLock) { + LogGatheringState state = logGatheringState; + if (state == LogGatheringState.NOT_GATHERING) { + return; + } + boolean gathering = state == LogGatheringState.GATHERING; + //while nothing is decided the wanted levels are unknown, so every level is held and filtered on adoption + if (gathering && logGatherLevels.indexOf(level) < 0) { + return; + } + + LogLine line = new LogLine(nextLineSequence++, System.currentTimeMillis(), level, trimLogMessage(msg), consentFeatures); + if (consentGiven) { + gatheredLines.addLast(line); + gatheredChars += line.message.length(); + trimBufferLocked(); + batchIsFull = gathering && gatheredLines.size() >= logGatherBatchSize; + } else { + linesAwaitingConsent.addLast(line); + awaitingConsentChars += line.message.length(); + trimAwaitingConsentLocked(); + } + } + + if (batchIsFull && !logDeliveryAwaitsSentBatch.get()) { + scheduleLogDelivery(false); + } else if (!consentGiven && isConsentGiven(consentFeatures)) { + //given since it was read above, possibly right after the release that would have moved this line + releaseConsentedLogLines(); + } + } + + /** + * Whether a line needing the given consent may join the gather now. Says no until the + * providers are set, so the lines logged before that wait. + * + * @param consentFeatures what {@link #logLineConsentFeatures(String)} returned for the line + * @return {@code true} when the line may join the gather + */ + private boolean isConsentGiven(int consentFeatures) { + IntSupplier consents = givenConsents; + return consents != null && isConsentGiven(consentFeatures, consents.getAsInt()); + } + + /** + * Whether a line needing the given consent may join the gather. + * + * @param consentFeatures the features any one of whose consent lets the line go, 0 for + * {@link #UNATTRIBUTED_LINE_CONSENT} + * @param given the {@link CoreFeature} indexes whose consent is given + * @return {@code true} when the line may join the gather + */ + static boolean isConsentGiven(int consentFeatures, int given) { + if (consentFeatures == 0) { + return (given & UNATTRIBUTED_LINE_CONSENT) == UNATTRIBUTED_LINE_CONSENT; + } + return (given & consentFeatures) != 0; + } + + /** + * Finds the consent a log line needs from the name it starts with in brackets, up to the + * closing bracket or a comma. + * + * @param message the line as logged + * @return the {@link CoreFeature} indexes any one of whose consent lets the line go, 0 for a + * line that needs {@link #UNATTRIBUTED_LINE_CONSENT} + */ + static int logLineConsentFeatures(@Nonnull String message) { + if (message.isEmpty() || message.charAt(0) != '[') { + return 0; + } + int limit = Math.min(message.length(), maxLogLineNameLength + 2); + int end = 1; + while (end < limit && message.charAt(end) != ']' && message.charAt(end) != ',') { + end++; + } + if (end == 1 || end == limit) { + return 0; + } + Integer features = logLineConsent.get(message.substring(1, end)); + return features == null ? 0 : features; + } + + /** + * Maps the bracketed names a feature's log lines start with to the consent of that feature. + * + * @param consent the map to fill + * @param feature the feature whose consent the lines need + * @param names the names, without the brackets + */ + private static void mapLogLineConsent(@Nonnull Map consent, @Nonnull CoreFeature feature, @Nonnull String... names) { + for (String name : names) { + consent.put(name, feature.getIndex()); + } + } + + /** + * Mirrors a directive and applies it to what is held. + * + * @param newState the decision + * @param newGatherId the gather id, {@code null} unless gathering + * @param newLevels the level characters to gather + * @param newBatchSize how many lines make a batch + * @return whether anything changed + */ + private boolean adoptLogGatheringDirective(@Nonnull LogGatheringState newState, @Nullable String newGatherId, @Nonnull String newLevels, int newBatchSize) { + synchronized (logBufferLock) { + if (logGatheringEnded) { + return false; + } + + boolean changed = logGatheringState != newState + || !Objects.equals(logGatherId, newGatherId) + || !logGatherLevels.equals(newLevels) + || logGatherBatchSize != newBatchSize; + if (!changed) { + return false; + } + + boolean wasGathering = logGatheringState == LogGatheringState.GATHERING; + String previousGatherId = logGatherId; + + logGatherId = newGatherId; + logGatherLevels = newLevels; + logGatherBatchSize = newBatchSize; + logGatheringState = newState; + + switch (newState) { + case NOT_GATHERING: + //the server accepts the last lines of a stopped gather under its id; lines held while undecided are dropped + if (wasGathering && previousGatherId != null && !gatheredLines.isEmpty()) { + pendingTailBatch = new LogBatch(previousGatherId, droppedLogLineCount, new ArrayList<>(gatheredLines)); + } + clearBufferLocked(); + break; + case GATHERING: + if (wasGathering && previousGatherId != null && !previousGatherId.equals(newGatherId)) { + //the held lines belong to the previous gather and never go out under another id + clearBufferLocked(); + } else { + filterBufferLocked(); + trimBufferLocked(); + } + break; + case UNDECIDED: + default: + break; + } + return true; + } + } + + /** + * Asks the delivery thread to queue what is held, never the calling thread, which may hold a + * lock of its own. A delivery already waiting to run takes the request over. Never logs, as it + * runs from the capture of a line. + * + * @param includePartialBatch whether a batch that is not full goes out too + */ + private void scheduleLogDelivery(boolean includePartialBatch) { + if (!sdkInitFinished) { + return; + } + if (includePartialBatch) { + partialLogDeliveryRequested.set(true); + } + if (!logDeliveryPending.compareAndSet(false, true)) { + return; + } + + ExecutorService executor = logDeliveryExecutor; + if (executor == null) { + logDeliveryPending.set(false); + return; + } + + try { + executor.execute(this::runLogDelivery); + } catch (RejectedExecutionException e) { + logDeliveryPending.set(false); + } + } + + /** + * Runs one delivery on the delivery thread: every full batch, and when asked also what is held + * beyond them, up to what was held when the delivery started. + */ + private void runLogDelivery() { + logDeliveryPending.set(false); + boolean includePartialBatch = partialLogDeliveryRequested.getAndSet(false); + try { + synchronized (logDeliveryLock) { + if (logGatheringEnded) { + return; + } + deliverHeldLogBatches(includePartialBatch ? heldLogLineCount() : 0, true); + } + } catch (RuntimeException e) { + e("[Log] runLogDelivery, failed to upload the gathered log lines, [" + e + "]"); + } + } + + /** + * Queues the held lines batch by batch, oldest first, until nothing is left to queue or the + * queue refuses a batch: every full batch, and within the given budget also a batch that is not + * full. Holds everything while tracking is off. The buffer holds only lines whose consent was + * given. The caller holds the delivery lock and has marked its thread. + * + * @param partialLineBudget how many lines may still go out in batches that are not full + * @param oneBatchAtATime whether to queue nothing while a batch this logger queued waits in the + * request queue, and at most one batch otherwise + */ + private void deliverHeldLogBatches(int partialLineBudget, boolean oneBatchAtATime) { + LogBatchQueue queue = logBatchQueue; + ConfigurationProvider provider = configProvider; + if (queue == null || provider == null) { + return; + } + + int budget = partialLineBudget; + while (true) { + if (!provider.getTrackingEnabled()) { + d("[Log] deliverHeldLogBatches, tracking is disabled, keeping the gathered lines buffered"); + return; + } + if (oneBatchAtATime) { + if (forgetLogBatchesNoLongerQueued(queue) > 0) { + if (hasLinesToDeliver(budget > 0)) { + logDeliveryAwaitsSentBatch.set(true); + d("[Log] deliverHeldLogBatches, a log batch is still waiting in the request queue, delivering the rest once it is sent"); + } + return; + } + //no batch waits any more, as when the queue dropped it unsent, so a resume asked for earlier is moot + logDeliveryAwaitsSentBatch.set(false); + } + + LogBatch batch = takeLogBatch(budget > 0); + if (batch == null) { + return; + } + budget -= batch.lines.size(); + + String payload = buildLogBatchPayload(batch.gatherId, batch.dropped, batch.lines); + d("[Log] deliverHeldLogBatches, uploading a gathered log batch of [" + batch.lines.size() + "] lines, [" + payload.length() + "] characters"); + if (!queueLogBatch(queue, payload)) { + d("[Log] deliverHeldLogBatches, the request queue refused the batch, keeping its lines buffered"); + restoreLogBatch(batch); + return; + } + } + } + + /** + * Hands one batch to the request queue and keeps the storage ID of its request, so the next + * batch waits while the queue holds it. + * + * @param queue the request queue + * @param payload the batch + * @return whether the queue took the batch + */ + private boolean queueLogBatch(@Nonnull LogBatchQueue queue, @Nonnull String payload) { + Long requestId = queue.queueLogBatch(payload); + if (requestId == null) { + return false; + } + queuedLogBatchIds.add(requestId); + return true; + } + + /** + * Forgets the batches the request queue no longer holds, sent or dropped, which also covers a + * request dropped without a send, as when the queue is over its size. + * + * @param queue the request queue + * @return how many batches of this logger the queue still holds + */ + private int forgetLogBatchesNoLongerQueued(@Nonnull LogBatchQueue queue) { + queuedLogBatchIds.removeIf(requestId -> !queue.isQueued(requestId)); + return queuedLogBatchIds.size(); + } + + /** + * Whether a delivery would queue anything now. + * + * @param includePartialBatch whether a batch that is not full counts + * @return {@code true} when a batch could be taken + */ + private boolean hasLinesToDeliver(boolean includePartialBatch) { + synchronized (logBufferLock) { + return canTakeLogBatchLocked(includePartialBatch); + } + } + + /** + * Whether a batch can be taken out of the buffer. The caller holds the buffer lock. + * + * @param includePartialBatch whether a batch that is not full counts + * @return {@code true} when a batch can be taken + */ + private boolean canTakeLogBatchLocked(boolean includePartialBatch) { + if (logGatheringState != LogGatheringState.GATHERING || logGatherId == null || gatheredLines.isEmpty()) { + return false; + } + return includePartialBatch || gatheredLines.size() >= logGatherBatchSize; + } + + /** + * Queues the last batch of a gather the server stopped, once, from the delivery thread, as the + * decision that stopped the gather may hold a lock the request queue needs. + */ + private void queueTailBatch() { + synchronized (logDeliveryLock) { + LogBatch tail = takePendingTailBatch(); + LogBatchQueue queue = logBatchQueue; + if (tail == null || queue == null || !sdkInitFinished || logGatheringEnded) { + return; + } + try { + queueTail(queue, tail); + } catch (RuntimeException e) { + e("[Log] queueTailBatch, failed to upload the last batch, [" + e + "]"); + } + } + } + + /** + * Takes the last batch of a gather the server stopped, if one waits. + * + * @return the batch, {@code null} for none + */ + @Nullable + private LogBatch takePendingTailBatch() { + synchronized (logBufferLock) { + LogBatch tail = pendingTailBatch; + pendingTailBatch = null; + return tail; + } + } + + /** + * Queues the last batch of a stopped gather, with its id. Its lines are gone if the queue + * refuses it. The caller holds the delivery lock. + * + * @param queue the request queue + * @param tail the batch + */ + private void queueTail(@Nonnull LogBatchQueue queue, @Nonnull LogBatch tail) { + d("[Log] queueTail, uploading the last [" + tail.lines.size() + "] lines of the stopped gather"); + if (!queueLogBatch(queue, buildLogBatchPayload(tail.gatherId, tail.dropped, tail.lines))) { + d("[Log] queueTail, the request queue refused the last batch, its lines are dropped"); + } + } + + /** + * Takes up to one batch of the oldest lines out of the buffer, with the lines dropped since the + * last batch, which are reported once, with the batch that follows the loss. + * + * @param includePartialBatch whether a batch that is not full may be taken + * @return the batch, {@code null} when there is nothing to take + */ + @Nullable + private LogBatch takeLogBatch(boolean includePartialBatch) { + synchronized (logBufferLock) { + if (!canTakeLogBatchLocked(includePartialBatch)) { + return null; + } + + int count = Math.min(logGatherBatchSize, gatheredLines.size()); + List taken = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + LogLine line = gatheredLines.removeFirst(); + gatheredChars -= line.message.length(); + taken.add(line); + } + + LogBatch batch = new LogBatch(logGatherId, droppedLogLineCount, taken); + droppedLogLineCount = 0; + return batch; + } + } + + /** + * Puts a batch the queue refused back at the front of the buffer, oldest first, with its drop + * count, unless its gather ended meanwhile. + * + * @param batch the refused batch + */ + private void restoreLogBatch(@Nonnull LogBatch batch) { + synchronized (logBufferLock) { + if (logGatheringState != LogGatheringState.GATHERING || !batch.gatherId.equals(logGatherId)) { + return; + } + + for (int i = batch.lines.size() - 1; i >= 0; i--) { + LogLine line = batch.lines.get(i); + if (logGatherLevels.indexOf(line.level) > -1) { + gatheredLines.addFirst(line); + gatheredChars += line.message.length(); + } + } + droppedLogLineCount += batch.dropped; + trimBufferLocked(); + } + } + + /** + * Creates the delivery thread's executor unless it exists or the gathering ended. The thread is + * a daemon, and it is marked as doing transport work for its whole life. + */ + private void ensureLogDeliveryExecutor() { + synchronized (logDeliveryExecutorLock) { + if (logGatheringEnded || logDeliveryExecutor != null) { + return; + } + logDeliveryExecutor = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(() -> { + setOwnTransportWork(true); + runnable.run(); + }, DELIVERY_THREAD_NAME); + thread.setDaemon(true); + return thread; + }); + } + v("[Log] ensureLogDeliveryExecutor, created the log batch delivery executor"); + } + + /** + * Releases the delivery thread once it queued the last batch of a gather the server stopped, + * if there is one. The batch is queued on the delivery thread, behind any delivery already + * waiting there, which finds nothing to do once the gathering is off. + */ + private void shutdownLogDeliveryAfterTail() { + ExecutorService executor = takeLogDeliveryExecutor(); + if (executor == null) { + takePendingTailBatch(); + return; + } + + try { + executor.execute(this::queueTailBatch); + } catch (RejectedExecutionException e) { + takePendingTailBatch(); + } + v("[Log] shutdownLogDeliveryAfterTail, stopping the log batch delivery executor"); + executor.shutdown(); + } + + /** + * Takes the delivery thread's executor away, so no delivery is scheduled on it any more. + * + * @return the executor, {@code null} when there was none + */ + @Nullable + private ExecutorService takeLogDeliveryExecutor() { + synchronized (logDeliveryExecutorLock) { + ExecutorService executor = logDeliveryExecutor; + logDeliveryExecutor = null; + return executor; + } + } + + /** + * Enforces both buffer ceilings by dropping the oldest lines, counting them into {@code d}. + */ + private void trimBufferLocked() { + while (gatheredLines.size() > ModuleConfiguration.logGatheringMaxBufferedLines + || (gatheredChars > gatheredCharCeiling && gatheredLines.size() > 1)) { + gatheredChars -= gatheredLines.removeFirst().message.length(); + droppedLogLineCount++; + } + } + + /** + * Enforces both ceilings on the lines waiting for their consent by dropping the oldest. They + * are not counted into {@code d}, as a line waiting for its consent has not joined the gather. + */ + private void trimAwaitingConsentLocked() { + while (linesAwaitingConsent.size() > ModuleConfiguration.logGatheringMaxBufferedLines + || (awaitingConsentChars > gatheredCharCeiling && linesAwaitingConsent.size() > 1)) { + awaitingConsentChars -= linesAwaitingConsent.removeFirst().message.length(); + } + } + + /** + * Adds lines whose consent was given to the buffer, which stays in capture order. + * + * @param released the lines, in capture order + */ + private void mergeIntoBufferLocked(@Nonnull List released) { + List merged = new ArrayList<>(gatheredLines); + merged.addAll(released); + merged.sort(Comparator.comparingLong(line -> line.sequence)); + for (LogLine line : released) { + gatheredChars += line.message.length(); + } + + gatheredLines.clear(); + gatheredLines.addAll(merged); + trimBufferLocked(); + } + + /** + * Drops the lines of levels the directive does not ask for, held or waiting for their consent. + * They are not counted as dropped, as they were never wanted. + */ + private void filterBufferLocked() { + Iterator iterator = gatheredLines.iterator(); + while (iterator.hasNext()) { + LogLine line = iterator.next(); + if (logGatherLevels.indexOf(line.level) < 0) { + gatheredChars -= line.message.length(); + iterator.remove(); + } + } + + Iterator awaiting = linesAwaitingConsent.iterator(); + while (awaiting.hasNext()) { + LogLine line = awaiting.next(); + if (logGatherLevels.indexOf(line.level) < 0) { + awaitingConsentChars -= line.message.length(); + awaiting.remove(); + } + } + } + + /** + * Empties the buffer and the lines waiting for their consent, and forgets the drop count. + */ + private void clearBufferLocked() { + gatheredLines.clear(); + gatheredChars = 0; + linesAwaitingConsent.clear(); + awaitingConsentChars = 0; + droppedLogLineCount = 0; + } + + /** + * Builds the {@code sdk_logs} value of one batch, its keys always in the same order. + * + * @param gatherId the gather id the directive named, which the server requires + * @param dropped how many lines were dropped since the last batch + * @param lines the lines, oldest first + * @return the JSON text + */ + @Nonnull + static String buildLogBatchPayload(@Nonnull String gatherId, int dropped, @Nonnull List lines) { + StringBuilder payload = new StringBuilder(64 + lines.size() * 96); + payload.append("{\"").append(keyBatchGatherId).append("\":").append(JSONObject.quote(gatherId)) + .append(",\"").append(keyBatchDroppedCount).append("\":").append(dropped) + .append(",\"").append(keyBatchLines).append("\":["); + for (int i = 0; i < lines.size(); i++) { + LogLine line = lines.get(i); + if (i > 0) { + payload.append(','); + } + payload.append("{\"").append(keyLineTimestamp).append("\":").append(line.timestamp) + .append(",\"").append(keyLineLevel).append("\":\"").append(line.level) + .append("\",\"").append(keyLineMessage).append("\":").append(JSONObject.quote(line.message)) + .append('}'); + } + return payload.append("]}").toString(); + } + + /** + * Cuts a message to what the server keeps, never between the two halves of a surrogate pair. + * + * @param message the message + * @return the message, at most {@value #maxMessageLength} characters long + */ + @Nonnull + static String trimLogMessage(@Nonnull String message) { + if (message.length() <= maxMessageLength) { + return message; + } + + int end = maxMessageLength; + if (Character.isHighSurrogate(message.charAt(end - 1))) { + end--; + } + return message.substring(0, end); + } + + /** + * One captured line. + */ + static final class LogLine { + final long sequence; + final long timestamp; + final char level; + final String message; + final int consentFeatures; + + /** + * Holds one line. + * + * @param sequence its place in capture order + * @param timestamp when it was logged, in milliseconds + * @param level the level character + * @param message the message, already cut + * @param consentFeatures the {@link CoreFeature} indexes any one of whose consent lets the + * line go, 0 for {@link #UNATTRIBUTED_LINE_CONSENT} + */ + LogLine(long sequence, long timestamp, char level, @Nonnull String message, int consentFeatures) { + this.sequence = sequence; + this.timestamp = timestamp; + this.level = level; + this.message = message; + this.consentFeatures = consentFeatures; + } + } + + /** + * One batch taken out of the buffer, kept whole so a refused batch can be put back. + */ + private static final class LogBatch { + final String gatherId; + final int dropped; + final List lines; + + /** + * Holds one batch. + * + * @param gatherId the gather id the batch is uploaded with + * @param dropped how many lines were dropped before it + * @param lines the lines, oldest first + */ + LogBatch(@Nonnull String gatherId, int dropped, @Nonnull List lines) { + this.gatherId = gatherId; + this.dropped = dropped; + this.lines = lines; + } + } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBackendMode.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBackendMode.java index 2dfc65d4..42e2256c 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBackendMode.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBackendMode.java @@ -79,7 +79,7 @@ private void recordEventInternal(String deviceID, String key, int count, Double eventQueues.get(deviceID).put(jsonObject); ++eventQSize; - if (eventQSize >= internalConfig.getEventsBufferSize()) { + if (eventQSize >= internalConfig.getConfigurationProvider().getEventQueueSizeThreshold()) { addEventsToRequestQ(); } } @@ -319,7 +319,7 @@ private synchronized void addEventsToRequestQ() { private void addRequestToRequestQ(Request request) { synchronized (SDKCore.instance.lockBRQStorage) { L.d("[BackendMode] addRequestToRequestQ"); - if (internalConfig.getRequestQueueMaxSize() == SDKCore.instance.requestQueueMemory.size()) { + if (internalConfig.getConfigurationProvider().getRequestQueueMaxSize() == SDKCore.instance.requestQueueMemory.size()) { L.d("[BackendMode] addRequestToRequestQ: In Memory request queue is full, dropping oldest request: " + request.params.toString()); SDKCore.instance.requestQueueMemory.remove(); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBase.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBase.java index 0b676ace..a6c786cb 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBase.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBase.java @@ -1,5 +1,7 @@ package ly.count.sdk.java.internal; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import ly.count.sdk.java.Session; /** @@ -13,6 +15,9 @@ * */ public abstract class ModuleBase { + //what a call through an interface while its module does not run hands to its callback + static final String NOT_RUNNING_ERROR = "The feature does not run, as its consent is not given or it is not enabled"; + private boolean active = false; protected Log L = null; @@ -47,6 +52,27 @@ public void init(InternalConfig config) { public void stop(InternalConfig config, boolean clear) { } + /** + * The instance of this module that the SDK runs, which every call through an interface of this + * module goes to: this one, or the running one when the interface was handed out before that + * started or kept from an instance that stopped. A call while none runs, as before init or while + * the consent of the feature is not given, is ignored and logged. + * + * @param call the call, starting with the bracketed name of the interface, for the log line + * @param the module type + * @return the running instance, {@code null} while none runs + */ + @SuppressWarnings("unchecked") + @Nullable + M runningInstance(@Nonnull String call) { + SDKCore core = SDKCore.instance; + ModuleBase running = core == null ? null : core.module(getClass()); + if (running == null && core != null && core.L != null) { + core.L.w(call + ", the feature does not run, as its consent is not given or it is not enabled, ignoring the call"); + } + return (M) running; + } + /** * A method to be used by the module itself to determine if {@link #init(InternalConfig)} initialized it * and haven't been stopped yet by {@link #stop(InternalConfig, boolean)}. @@ -95,9 +121,16 @@ public Boolean onRequest(Request request) { } /** - * Called when the request is executed. + * Called when the request is executed, only on the owning module ({@link Request#own(Class)}), + * once for every attempt to send it, on the networking thread. A request dropped unsent for + * being older than the request drop age gets a call too, with a {@code null} response and + * {@link Transport#NO_RESPONSE_CODE}. * Gives the module the ability to respond to the response received * Request identification should be done through the Request ID + * + * @param request the request that was sent + * @param response the response body, {@code null} when none could be read + * @param responseCode the response code, {@link Transport#NO_RESPONSE_CODE} when no response arrived */ public void onRequestCompleted(Request request, String response, int responseCode) { @@ -124,4 +157,17 @@ protected void initFinished(InternalConfig config) { protected void deviceIdChanged(String oldDeviceId, boolean withMerge) { } + + /** + * Called when a server response changed the SDK behavior settings while the SDK runs, after + * the SDK applied what it owns: the logger, the timer interval and the request queue. Read the + * new values through {@link InternalConfig#getConfigurationProvider()}. Runs on the thread that + * delivered the response, which is usually not the thread that initialized the SDK, and can run + * after {@link #init(InternalConfig)} but before {@link #initFinished(InternalConfig)} when the + * response of the fetch made at init arrives that early. + * + * @param config configuration of the running SDK + */ + protected void onSdkConfigurationChanged(InternalConfig config) { + } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java new file mode 100644 index 00000000..58f6ba70 --- /dev/null +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java @@ -0,0 +1,1317 @@ +package ly.count.sdk.java.internal; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import ly.count.sdk.java.Config; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Resolves, stores and serves the SDK behavior settings, the server side configuration served by + * {@code /o/sdk?method=sc}. + *

+ * Every setting starts from the developer configuration and is overridden by the settings provided + * at init (only while nothing is stored), by the stored settings and by every later server response. + * Each accepted response replaces the stored settings as a whole, so a setting the server stops + * sending goes back to the developer configuration, or to the SDK default when it sets none. + * {@link SDKCore} runs {@link #init(InternalConfig)}, which does the resolve, before it starts the + * global timer and builds the other modules; {@link #initFinished(InternalConfig)} fetches the + * settings and starts the refresh timer. In backend mode the module stays inert and serves the + * developer configuration. + *

+ * Resolved values are published through volatile fields and immutable snapshots, so a reader on any + * thread sees consistent values while a response is applied on the thread that delivered it. + */ +public class ModuleConfiguration extends ModuleBase implements ConfigurationProvider { + + static final String keyRTimestamp = "t"; + static final String keyRVersion = "v"; + static final String keyRConfig = "c"; + static final String keyRLogGathering = "lg"; + static final String keyRConnectionTest = "ct"; + + static final String keyRTracking = "tracking"; + static final String keyRNetworking = "networking"; + static final String keyRReqQueueSize = "rqs"; + static final String keyREventQueueSize = "eqs"; + static final String keyRLogging = "log"; + static final String keyRSessionUpdateInterval = "sui"; + static final String keyRSessionTracking = "st"; + static final String keyRViewTracking = "vt"; + static final String keyRLocationTracking = "lt"; + static final String keyRRefreshContentZone = "rcz"; + static final String keyRLimitKeyLength = "lkl"; + static final String keyRLimitValueSize = "lvs"; + static final String keyRLimitSegValues = "lsv"; + static final String keyRLimitBreadcrumb = "lbc"; + static final String keyRLimitTraceLine = "ltlpt"; + static final String keyRLimitTraceLength = "ltl"; + static final String keyRCustomEventTracking = "cet"; + static final String keyREnterContentZone = "ecz"; + static final String keyRContentZoneInterval = "czi"; + static final String keyRConsentRequired = "cr"; + static final String keyRDropOldRequestTime = "dort"; + static final String keyRCrashReporting = "crt"; + static final String keyRAutomaticSessionTracking = "ast"; + static final String keyRAutomaticViewTracking = "avt"; + static final String keyRAutomaticCrashReporting = "acr"; + static final String keyRServerConfigUpdateInterval = "scui"; + static final String keyRBackoffMechanism = "bom"; + static final String keyRBOMAcceptedTimeout = "bom_at"; + static final String keyRBOMRQPercentage = "bom_rqp"; + static final String keyRBOMRequestAge = "bom_ra"; + static final String keyRBOMDuration = "bom_d"; + static final String keyRUserPropertyCacheLimit = "upcl"; + static final String keyREventBlacklist = "eb"; + static final String keyRUserPropertyBlacklist = "upb"; + static final String keyRSegmentationBlacklist = "sb"; + static final String keyREventSegmentationBlacklist = "esb"; + static final String keyREventWhitelist = "ew"; + static final String keyRUserPropertyWhitelist = "upw"; + static final String keyRSegmentationWhitelist = "sw"; + static final String keyREventSegmentationWhitelist = "esw"; + static final String keyRJourneyTriggerEvents = "jte"; + static final String keyRJourneyTriggerViews = "jtv"; + + static final String keyLGEnabled = "e"; + static final String keyLGId = "i"; + static final String keyLGLevels = "l"; + static final String keyLGBatchSize = "b"; + + static final String logGatheringAllLevels = "ewidv"; + static final int logGatheringDefaultBatchSize = 100; + static final int logGatheringMinBatchSize = 10; + //the log buffer ceiling, so also the largest batch the server may ask for: a bigger batch would never fill + static final int logGatheringMaxBufferedLines = 500; + + static final int DEFAULT_SERVER_CONFIG_UPDATE_INTERVAL_HOURS = 4; + static final int DEFAULT_BOM_ACCEPTED_TIMEOUT_SECONDS = 10; + static final double DEFAULT_BOM_RQ_PERCENTAGE = 0.5; + static final int DEFAULT_BOM_REQUEST_AGE_HOURS = 24; + static final int DEFAULT_BOM_DURATION_SECONDS = 60; + static final int MIN_CONTENT_ZONE_INTERVAL = 16; + static final int NO_LIMIT = Integer.MAX_VALUE; + + //the settings of the last accepted configuration; guarded by this + JSONObject latestRetrievedConfiguration = null; + + //what each setting goes back to when the stored settings do not carry it, read from the developer configuration at init + private ConfigurationProvider developerValues = null; + + //guarded by this + CountlyTimer serverConfigUpdateTimer = null; + + private boolean serverConfigRequestsDisabled = false; + private volatile boolean running = false; + /** + * How many times the device ID changed without merge, written under this module's lock, so a + * response requested before such a change can tell its gather was armed for the previous user. + */ + private volatile int deviceIdChangesWithoutMerge = 0; + volatile ConnectionTestListener connectionTestListener = null; + + //resolved values; the constant seeds are here, the ones read from the developer configuration are in seedFromDeveloperConfig + volatile boolean currentVTracking = true; + volatile boolean currentVNetworking = true; + volatile boolean currentVSessionTracking = true; + volatile boolean currentVViewTracking = true; + volatile boolean currentVCustomEventTracking = true; + volatile boolean currentVContentZone = false; + volatile boolean currentVCrashReporting = true; + volatile boolean currentVAutomaticSessionTracking = false; + volatile boolean currentVAutomaticViewTracking = false; + volatile boolean currentVAutomaticCrashReporting = true; + volatile boolean currentVLocationTracking = true; + volatile boolean currentVRefreshContentZone = true; + volatile boolean currentVBackoffMechanism = true; + volatile boolean currentVLoggingEnabled = false; + volatile boolean currentVRequiresConsent = false; + + volatile int currentVServerConfigUpdateInterval = DEFAULT_SERVER_CONFIG_UPDATE_INTERVAL_HOURS; + volatile int currentVRequestQueueMaxSize = 1000; + volatile boolean currentVRequestQueueMaxSizeFromBehaviorSettings = false; + volatile int currentVEventQueueSizeThreshold = 10; + volatile int currentVSessionUpdateInterval = 60; + volatile int currentVMaxKeyLength = NO_LIMIT; + volatile int currentVMaxValueSize = NO_LIMIT; + volatile int currentVMaxSegmentationValues = NO_LIMIT; + volatile int currentVMaxBreadcrumbCount = 100; + volatile int currentVMaxStackTraceLinesPerThread = NO_LIMIT; + volatile int currentVMaxStackTraceLineLength = NO_LIMIT; + volatile int currentVUserPropertyCacheLimit = NO_LIMIT; + volatile int currentVBOMAcceptedTimeoutSeconds = DEFAULT_BOM_ACCEPTED_TIMEOUT_SECONDS; + volatile double currentVBOMRQPercentage = DEFAULT_BOM_RQ_PERCENTAGE; + volatile int currentVBOMRequestAge = DEFAULT_BOM_REQUEST_AGE_HOURS; + volatile int currentVBOMDuration = DEFAULT_BOM_DURATION_SECONDS; + volatile int currentVDropAgeHours = 0; + volatile int currentVZoneTimerInterval = ConfigContent.DEFAULT_ZONE_TIMER_INTERVAL; + + volatile FilterList> currentVEventFilterList = FilterList.NO_NAMES; + volatile FilterList> currentVUserPropertyFilterList = FilterList.NO_NAMES; + volatile FilterList> currentVSegmentationFilterList = FilterList.NO_NAMES; + volatile FilterList>> currentVEventSegmentationFilterList = FilterList.NO_NAMES_PER_EVENT; + volatile Set currentVJourneyTriggerEvents = Collections.emptySet(); + volatile Set currentVJourneyTriggerViews = Collections.emptySet(); + volatile LogGatheringDirective currentVLogGathering = LogGatheringDirective.UNDECIDED; + + /** + * Runs the connection test that a live settings response asks for with its {@code ct} flag. + */ + interface ConnectionTestListener { + /** + * Called on the thread that delivered the response, after the settings in it were applied. + * + * @param fetchLatencyMs how long the settings request took, from sending it to its response, in milliseconds + */ + void onConnectionTestRequested(long fetchLatencyMs); + } + + /** + * One decision of the log gathering directive, published as a whole so a reader never sees the + * state of one directive next to the gather id of another. + */ + static final class LogGatheringDirective { + static final LogGatheringDirective UNDECIDED = new LogGatheringDirective(LogGatheringState.UNDECIDED, null, logGatheringAllLevels, logGatheringDefaultBatchSize); + static final LogGatheringDirective NOT_GATHERING = new LogGatheringDirective(LogGatheringState.NOT_GATHERING, null, logGatheringAllLevels, logGatheringDefaultBatchSize); + + final LogGatheringState state; + final String gatherId; + final String levels; + final int batchSize; + + /** + * Holds one decision. + * + * @param state what was decided + * @param gatherId the gather id, {@code null} unless gathering + * @param levels the level characters to gather + * @param batchSize how many lines make a batch + */ + LogGatheringDirective(@Nonnull LogGatheringState state, @Nullable String gatherId, @Nonnull String levels, int batchSize) { + this.state = state; + this.gatherId = gatherId; + this.levels = levels; + this.batchSize = batchSize; + } + } + + /** + * Builds a provider that serves the developer configuration alone, for code that reads the + * settings while no configuration module is registered on that configuration. + * + * @param config the developer configuration to serve + * @return a provider that never changes + */ + @Nonnull + static ConfigurationProvider developerDefaults(@Nonnull InternalConfig config) { + ModuleConfiguration defaults = new ModuleConfiguration(); + defaults.seedFromDeveloperConfig(config); + return defaults; + } + + /** + * Seeds every setting from the developer configuration and resolves the provided and the stored + * settings over it, then registers this module as the provider of the configuration. In backend + * mode nothing is loaded and the developer configuration stays in effect. + * + * @param config configuration of the SDK being initialized + */ + @Override + public void init(InternalConfig config) { + super.init(config); + boolean backendMode = config.isBackendModeEnabled(); + serverConfigRequestsDisabled = backendMode || config.isSdkBehaviorSettingsRequestsDisabled(); + L.d("[ModuleConfiguration] init, backend mode:[" + backendMode + "], server config requests disabled:[" + serverConfigRequestsDisabled + "]"); + + seedFromDeveloperConfig(config); + developerValues = developerDefaults(config); + + if (backendMode) { + L.d("[ModuleConfiguration] init, backend mode is enabled, SDK behavior settings are not applied"); + } else { + loadConfigFromStorage(config.getSdkBehaviorSettings()); + } + + updateConfigVariables(config, null, true); + running = true; + config.configProvider = this; + } + + /** + * Fetches the settings from the server and starts the refresh timer, unless settings requests + * are disabled or backend mode is on. + * + * @param config configuration of the running SDK + */ + @Override + public void initFinished(@Nonnull InternalConfig config) { + L.d("[ModuleConfiguration] initFinished"); + if (serverConfigRequestsDisabled) { + return; + } + + fetchConfigFromServer(config); + startServerConfigUpdateTimer(); + } + + /** + * Stops the refresh timer; a response that arrives afterwards is ignored. + * + * @param config configuration of the SDK being stopped + * @param clear whether the SDK clears its data, which removes the stored settings with it + */ + @Override + public void stop(InternalConfig config, boolean clear) { + super.stop(config, clear); + running = false; + stopServerConfigUpdateTimer(); + } + + /** + * Sets the resolved values that come from the developer configuration. + * + * @param config the developer configuration + */ + private void seedFromDeveloperConfig(@Nonnull InternalConfig config) { + currentVLoggingEnabled = config.getLoggingLevel() != null && config.getLoggingLevel() != Config.LoggingLevel.OFF; + currentVRequiresConsent = config.requiresConsent(); + currentVAutomaticCrashReporting = config.isUnhandledCrashReportingEnabled(); + currentVBackoffMechanism = config.isBackoffMechanismEnabled(); + currentVRequestQueueMaxSize = config.getRequestQueueMaxSize(); + currentVEventQueueSizeThreshold = config.getEventsBufferSize(); + currentVSessionUpdateInterval = config.getSendUpdateEachSeconds(); + currentVMaxBreadcrumbCount = config.getMaxBreadcrumbCount(); + currentVZoneTimerInterval = config.content.zoneTimerInterval; + } + + /** + * Loads the stored settings, or the provided ones when nothing is stored yet, which stores them. + * + * @param sdkBehaviorSettings the settings the developer provided, {@code null} for none + */ + void loadConfigFromStorage(@Nullable String sdkBehaviorSettings) { + String sConfig = internalConfig.storageProvider.getServerConfig(); + + if (Utils.isEmptyOrNull(sConfig) && !Utils.isEmptyOrNull(sdkBehaviorSettings)) { + L.d("[ModuleConfiguration] loadConfigFromStorage, nothing is stored, using the provided settings"); + sConfig = sdkBehaviorSettings; + } + + L.v("[ModuleConfiguration] loadConfigFromStorage, [" + sConfig + "]"); + + if (Utils.isEmptyOrNull(sConfig)) { + L.d("[ModuleConfiguration] loadConfigFromStorage, no configuration is stored or provided"); + return; + } + + try { + saveAndStoreDownloadedConfig(new JSONObject(sConfig)); + } catch (JSONException e) { + L.w("[ModuleConfiguration] loadConfigFromStorage, failed to parse, " + e); + } + } + + /** + * Requires {@code v}, {@code t} and an object {@code c}, then removes the settings this SDK does + * not support. Other top level keys are allowed; they are never stored. + * + * @param config a configuration as the server sends it, its settings object is changed in place + * @return whether the configuration can be stored + */ + boolean validateServerConfig(@Nonnull JSONObject config) { + L.v("[ModuleConfiguration] validateServerConfig"); + if (!config.has(keyRVersion)) { + L.w("[ModuleConfiguration] validateServerConfig, retrieved configuration has no 'v' field, it is ignored"); + return false; + } + if (!config.has(keyRTimestamp)) { + L.w("[ModuleConfiguration] validateServerConfig, retrieved configuration has no 't' field, it is ignored"); + return false; + } + if (!config.has(keyRConfig)) { + L.w("[ModuleConfiguration] validateServerConfig, retrieved configuration has no 'c' field, it is ignored"); + return false; + } + + JSONObject newInner = config.optJSONObject(keyRConfig); + if (newInner == null) { + L.w("[ModuleConfiguration] validateServerConfig, retrieved configuration has a 'c' that is not an object, it is ignored"); + return false; + } + + removeUnsupportedKeys(newInner); + return true; + } + + /** + * Removes every setting whose key is unknown or whose value has the wrong type or range. + * + * @param newInner the settings object, changed in place + */ + private void removeUnsupportedKeys(@Nonnull JSONObject newInner) { + Iterator keys = newInner.keys(); + while (keys.hasNext()) { + String key = keys.next(); + Object value = newInner.opt(key); + boolean isValid; + + switch (key) { + case keyRNetworking: + case keyRTracking: + case keyRSessionTracking: + case keyRCrashReporting: + case keyRAutomaticSessionTracking: + case keyRAutomaticViewTracking: + case keyRAutomaticCrashReporting: + case keyRViewTracking: + case keyRCustomEventTracking: + case keyRLocationTracking: + case keyREnterContentZone: + case keyRRefreshContentZone: + case keyRBackoffMechanism: + case keyRLogging: + case keyRConsentRequired: + isValid = value instanceof Boolean; + break; + case keyRServerConfigUpdateInterval: + case keyRBOMAcceptedTimeout: + case keyRBOMRequestAge: + case keyRBOMDuration: + case keyRReqQueueSize: + case keyREventQueueSize: + case keyRSessionUpdateInterval: + case keyRLimitKeyLength: + case keyRLimitValueSize: + case keyRLimitSegValues: + case keyRLimitBreadcrumb: + case keyRLimitTraceLine: + case keyRLimitTraceLength: + case keyRUserPropertyCacheLimit: + isValid = isIntAtLeast(value, 1); + break; + case keyRDropOldRequestTime: + isValid = isIntAtLeast(value, 0); + break; + case keyRContentZoneInterval: + isValid = isIntAtLeast(value, MIN_CONTENT_ZONE_INTERVAL); + break; + case keyRBOMRQPercentage: + isValid = isFraction(value); + break; + case keyREventBlacklist: + case keyRSegmentationBlacklist: + case keyRUserPropertyBlacklist: + case keyREventWhitelist: + case keyRSegmentationWhitelist: + case keyRUserPropertyWhitelist: + case keyRJourneyTriggerEvents: + case keyRJourneyTriggerViews: + isValid = value instanceof JSONArray; + break; + case keyREventSegmentationBlacklist: + case keyREventSegmentationWhitelist: + isValid = value instanceof JSONObject; + break; + default: + L.w("[ModuleConfiguration] removeUnsupportedKeys, unknown key [" + key + "], removing it, value: [" + value + "]"); + keys.remove(); + continue; + } + + if (!isValid) { + L.w("[ModuleConfiguration] removeUnsupportedKeys, invalid value for key [" + key + "], removing it, value: [" + value + "]"); + keys.remove(); + } + } + } + + /** + * Whether a setting value is an integer of at least the given value. + * + * @param value the value as parsed from JSON + * @param minimum the smallest valid value + * @return {@code true} for a valid value + */ + private static boolean isIntAtLeast(@Nullable Object value, int minimum) { + return value instanceof Integer && (Integer) value >= minimum; + } + + /** + * Whether a setting value is a number strictly between 0 and 1. Decimals are parsed as + * {@link java.math.BigDecimal}, so any {@link Number} is accepted. + * + * @param value the value as parsed from JSON + * @return {@code true} for a valid value + */ + private static boolean isFraction(@Nullable Object value) { + if (!(value instanceof Number)) { + return false; + } + double fraction = ((Number) value).doubleValue(); + return fraction > 0.0 && fraction < 1.0; + } + + /** + * Validates a configuration and replaces the stored settings with its settings, so a setting it + * does not carry is no longer stored. Only {@code v}, {@code t} and {@code c} are stored, so the + * log gathering directive and the connection test flag never are. An invalid configuration + * leaves the stored settings as they are. + * + * @param config a configuration as the server sends it + */ + synchronized void saveAndStoreDownloadedConfig(@Nonnull JSONObject config) { + L.v("[ModuleConfiguration] saveAndStoreDownloadedConfig"); + if (!validateServerConfig(config)) { + L.w("[ModuleConfiguration] saveAndStoreDownloadedConfig, retrieved configuration is not valid, ignoring it"); + return; + } + + List notStoredTopLevelKeys = new ArrayList<>(); + for (String key : config.keySet()) { + if (!keyRVersion.equals(key) && !keyRTimestamp.equals(key) && !keyRConfig.equals(key)) { + notStoredTopLevelKeys.add(key); + } + } + if (!notStoredTopLevelKeys.isEmpty()) { + L.d("[ModuleConfiguration] saveAndStoreDownloadedConfig, top level keys that are not stored: " + notStoredTopLevelKeys); + } + + //validation already removed every value that is not a usable setting, so the settings object is kept as it is + latestRetrievedConfiguration = config.optJSONObject(keyRConfig); + internalConfig.storageProvider.setServerConfig(new JSONObject(config, keyRVersion, keyRTimestamp, keyRConfig).toString()); + } + + /** + * Applies the stored settings to the resolved values, decides the log gathering directive of a + * live response requested since the last device ID change without merge and has the logger apply + * it, and restarts the refresh timer when its interval changed. A response requested before that + * change leaves the directive as it is, as its gather was armed for the previous user. + * + * @param config configuration of the running SDK + * @param serverResponse the live response being applied, {@code null} at init, where only stored or provided settings exist + * @param forCurrentDeviceId whether the response was requested since the last device ID change without merge + * @return whether a resolved value changed + */ + synchronized boolean updateConfigVariables(@Nonnull InternalConfig config, @Nullable JSONObject serverResponse, boolean forCurrentDeviceId) { + L.v("[ModuleConfiguration] updateConfigVariables, from server response:[" + (serverResponse != null) + "]"); + + if (forCurrentDeviceId) { + //read off the live response itself, so it applies even when its settings object was rejected + readLogGatheringDirective(serverResponse); + L.applyLogGatheringDirective(serverResponse == null ? "init" : "server response"); + } else { + L.d("[ModuleConfiguration] updateConfigVariables, the response was requested for a previous device ID, its log gathering directive is ignored"); + } + + if (latestRetrievedConfiguration == null) { + return false; + } + + StringBuilder sb = new StringBuilder(); + boolean previousRequiresConsent = currentVRequiresConsent; + ConfigurationProvider developer = developerValues; + + currentVNetworking = extractBoolean(keyRNetworking, sb, currentVNetworking, developer.getNetworkingEnabled()); + currentVTracking = extractBoolean(keyRTracking, sb, currentVTracking, developer.getTrackingEnabled()); + currentVSessionTracking = extractBoolean(keyRSessionTracking, sb, currentVSessionTracking, developer.getSessionTrackingEnabled()); + currentVCrashReporting = extractBoolean(keyRCrashReporting, sb, currentVCrashReporting, developer.getCrashReportingEnabled()); + currentVAutomaticSessionTracking = extractBoolean(keyRAutomaticSessionTracking, sb, currentVAutomaticSessionTracking, developer.getAutomaticSessionTrackingEnabled()); + currentVAutomaticViewTracking = extractBoolean(keyRAutomaticViewTracking, sb, currentVAutomaticViewTracking, developer.getAutomaticViewTrackingEnabled()); + currentVAutomaticCrashReporting = extractBoolean(keyRAutomaticCrashReporting, sb, currentVAutomaticCrashReporting, developer.getAutomaticCrashReportingEnabled()); + currentVViewTracking = extractBoolean(keyRViewTracking, sb, currentVViewTracking, developer.getViewTrackingEnabled()); + currentVCustomEventTracking = extractBoolean(keyRCustomEventTracking, sb, currentVCustomEventTracking, developer.getCustomEventTrackingEnabled()); + currentVLocationTracking = extractBoolean(keyRLocationTracking, sb, currentVLocationTracking, developer.getLocationTrackingEnabled()); + currentVContentZone = extractBoolean(keyREnterContentZone, sb, currentVContentZone, developer.getContentZoneEnabled()); + currentVRefreshContentZone = extractBoolean(keyRRefreshContentZone, sb, currentVRefreshContentZone, developer.getRefreshContentZoneEnabled()); + currentVBackoffMechanism = extractBoolean(keyRBackoffMechanism, sb, currentVBackoffMechanism, developer.getBOMEnabled()); + currentVLoggingEnabled = extractBoolean(keyRLogging, sb, currentVLoggingEnabled, developer.getLoggingEnabled()); + //the settings can turn the consent requirement on, never off when the developer turned it on + currentVRequiresConsent = developer.getConsentRequired() || extractBoolean(keyRConsentRequired, sb, currentVRequiresConsent, false); + + currentVServerConfigUpdateInterval = extractInt(keyRServerConfigUpdateInterval, sb, currentVServerConfigUpdateInterval, developer.getServerConfigUpdateInterval()); + currentVRequestQueueMaxSize = extractInt(keyRReqQueueSize, sb, currentVRequestQueueMaxSize, developer.getRequestQueueMaxSize()); + currentVEventQueueSizeThreshold = extractInt(keyREventQueueSize, sb, currentVEventQueueSizeThreshold, developer.getEventQueueSizeThreshold()); + currentVSessionUpdateInterval = extractInt(keyRSessionUpdateInterval, sb, currentVSessionUpdateInterval, developer.getSessionUpdateInterval()); + currentVMaxKeyLength = extractInt(keyRLimitKeyLength, sb, currentVMaxKeyLength, developer.getMaxKeyLength()); + currentVMaxValueSize = extractInt(keyRLimitValueSize, sb, currentVMaxValueSize, developer.getMaxValueSize()); + currentVMaxSegmentationValues = extractInt(keyRLimitSegValues, sb, currentVMaxSegmentationValues, developer.getMaxSegmentationValues()); + currentVMaxBreadcrumbCount = extractInt(keyRLimitBreadcrumb, sb, currentVMaxBreadcrumbCount, developer.getMaxBreadcrumbCount()); + currentVMaxStackTraceLinesPerThread = extractInt(keyRLimitTraceLine, sb, currentVMaxStackTraceLinesPerThread, developer.getMaxStackTraceLinesPerThread()); + currentVMaxStackTraceLineLength = extractInt(keyRLimitTraceLength, sb, currentVMaxStackTraceLineLength, developer.getMaxStackTraceLineLength()); + currentVUserPropertyCacheLimit = extractInt(keyRUserPropertyCacheLimit, sb, currentVUserPropertyCacheLimit, developer.getUserPropertyCacheLimit()); + currentVBOMAcceptedTimeoutSeconds = extractInt(keyRBOMAcceptedTimeout, sb, currentVBOMAcceptedTimeoutSeconds, developer.getBOMAcceptedTimeoutSeconds()); + currentVBOMRQPercentage = extractFraction(keyRBOMRQPercentage, sb, currentVBOMRQPercentage, developer.getBOMRQPercentage()); + currentVBOMRequestAge = extractInt(keyRBOMRequestAge, sb, currentVBOMRequestAge, developer.getBOMRequestAge()); + currentVBOMDuration = extractInt(keyRBOMDuration, sb, currentVBOMDuration, developer.getBOMDuration()); + currentVDropAgeHours = extractInt(keyRDropOldRequestTime, sb, currentVDropAgeHours, developer.getRequestDropAgeHours()); + currentVZoneTimerInterval = extractInt(keyRContentZoneInterval, sb, currentVZoneTimerInterval, developer.getContentZoneTimerInterval()); + currentVRequestQueueMaxSizeFromBehaviorSettings = latestRetrievedConfiguration.has(keyRReqQueueSize); + + updateListingFilters(); + + if (serverResponse != null && previousRequiresConsent != currentVRequiresConsent) { + L.d("[ModuleConfiguration] updateConfigVariables, consent requirement changed to [" + currentVRequiresConsent + "], it applies on the next init, the running SDK keeps [" + config.requiresConsent() + "]"); + } + + if (serverConfigUpdateTimer != null && serverConfigUpdateTimer.getTimerDelaySeconds() != serverConfigUpdateIntervalSeconds()) { + startServerConfigUpdateTimer(); + } + + String updatedValues = sb.toString(); + if (updatedValues.isEmpty()) { + return false; + } + + L.i("[ModuleConfiguration] updateConfigVariables, SDK configuration has changed, new values: [" + updatedValues + "]"); + return true; + } + + /** + * Resolves a switch from the stored settings. Validation keeps only usable values, so a stored + * value is used as it is. + * + * @param key the settings key + * @param changedValues where a change is described + * @param currentValue the value in effect + * @param fallbackValue the developer's value, or the SDK default when the developer sets none + * @return the stored value, or the fallback value when none is stored + */ + private boolean extractBoolean(@Nonnull String key, @Nonnull StringBuilder changedValues, boolean currentValue, boolean fallbackValue) { + Object value = latestRetrievedConfiguration.opt(key); + return noteChange(key, changedValues, currentValue, value instanceof Boolean ? (Boolean) value : fallbackValue); + } + + /** + * Resolves an integer from the stored settings. Validation keeps only usable values, so a + * stored value is used as it is. + * + * @param key the settings key + * @param changedValues where a change is described + * @param currentValue the value in effect + * @param fallbackValue the developer's value, or the SDK default when the developer sets none + * @return the stored value, or the fallback value when none is stored + */ + private int extractInt(@Nonnull String key, @Nonnull StringBuilder changedValues, int currentValue, int fallbackValue) { + Object value = latestRetrievedConfiguration.opt(key); + return noteChange(key, changedValues, currentValue, value instanceof Integer ? (Integer) value : fallbackValue); + } + + /** + * Resolves a fraction from the stored settings. Validation keeps only usable values, so a + * stored value is used as it is. + * + * @param key the settings key + * @param changedValues where a change is described + * @param currentValue the value in effect + * @param fallbackValue the SDK default + * @return the stored value, or the fallback value when none is stored + */ + private double extractFraction(@Nonnull String key, @Nonnull StringBuilder changedValues, double currentValue, double fallbackValue) { + Object value = latestRetrievedConfiguration.opt(key); + return noteChange(key, changedValues, currentValue, value instanceof Number ? ((Number) value).doubleValue() : fallbackValue); + } + + /** + * Describes a resolved value that differs from the value in effect. + * + * @param key the settings key + * @param changedValues where a change is described + * @param currentValue the value in effect + * @param resolvedValue the value just resolved + * @param the type of the value + * @return the resolved value + */ + private static T noteChange(@Nonnull String key, @Nonnull StringBuilder changedValues, @Nonnull T currentValue, @Nonnull T resolvedValue) { + if (!resolvedValue.equals(currentValue)) { + changedValues.append(key).append(":[").append(resolvedValue).append("], "); + } + return resolvedValue; + } + + /** + * Rebuilds the listing filters and the journey trigger sets from the stored settings and + * publishes each as a new immutable snapshot. A filter the stored settings do not hold is an + * empty blacklist, which allows everything. + */ + private void updateListingFilters() { + currentVEventFilterList = namesFilterOf(keyREventBlacklist, keyREventWhitelist); + currentVUserPropertyFilterList = namesFilterOf(keyRUserPropertyBlacklist, keyRUserPropertyWhitelist); + currentVSegmentationFilterList = namesFilterOf(keyRSegmentationBlacklist, keyRSegmentationWhitelist); + currentVEventSegmentationFilterList = namesPerEventFilterOf(keyREventSegmentationBlacklist, keyREventSegmentationWhitelist); + currentVJourneyTriggerEvents = Collections.unmodifiableSet(namesOf(latestRetrievedConfiguration.optJSONArray(keyRJourneyTriggerEvents))); + currentVJourneyTriggerViews = Collections.unmodifiableSet(namesOf(latestRetrievedConfiguration.optJSONArray(keyRJourneyTriggerViews))); + + L.d("[ModuleConfiguration] updateListingFilters, event filter:[" + currentVEventFilterList + + "], user property filter:[" + currentVUserPropertyFilterList + + "], segmentation filter:[" + currentVSegmentationFilterList + + "], event segmentation filter:[" + currentVEventSegmentationFilterList + + "], journey trigger events:" + currentVJourneyTriggerEvents + + ", journey trigger views:" + currentVJourneyTriggerViews); + } + + /** + * Builds a names filter from the stored settings, the blacklist winning when both are stored. + * + * @param blacklistKey the settings key of the blacklist + * @param whitelistKey the settings key of the whitelist + * @return the filter + */ + @Nonnull + private FilterList> namesFilterOf(@Nonnull String blacklistKey, @Nonnull String whitelistKey) { + JSONArray blacklist = latestRetrievedConfiguration.optJSONArray(blacklistKey); + if (blacklist != null) { + return FilterList.ofNames(namesOf(blacklist), false); + } + + JSONArray whitelist = latestRetrievedConfiguration.optJSONArray(whitelistKey); + if (whitelist != null) { + return FilterList.ofNames(namesOf(whitelist), true); + } + + return FilterList.NO_NAMES; + } + + /** + * Builds a names per event filter from the stored settings, the blacklist winning when both are stored. + * + * @param blacklistKey the settings key of the blacklist + * @param whitelistKey the settings key of the whitelist + * @return the filter + */ + @Nonnull + private FilterList>> namesPerEventFilterOf(@Nonnull String blacklistKey, @Nonnull String whitelistKey) { + JSONObject blacklist = latestRetrievedConfiguration.optJSONObject(blacklistKey); + if (blacklist != null) { + return FilterList.ofNamesPerEvent(namesPerEventOf(blacklist), false); + } + + JSONObject whitelist = latestRetrievedConfiguration.optJSONObject(whitelistKey); + if (whitelist != null) { + return FilterList.ofNamesPerEvent(namesPerEventOf(whitelist), true); + } + + return FilterList.NO_NAMES_PER_EVENT; + } + + /** + * Reads the names of a JSON array, skipping nulls and turning other values into strings. + * + * @param jsonArray the array, {@code null} for none + * @return the names in array order + */ + @Nonnull + private static Set namesOf(@Nullable JSONArray jsonArray) { + Set names = new LinkedHashSet<>(); + if (jsonArray == null) { + return names; + } + + for (int i = 0; i < jsonArray.length(); i++) { + String item = jsonArray.optString(i, null); + if (item != null) { + names.add(item); + } + } + return names; + } + + /** + * Reads the names of each event key of a JSON object, skipping keys whose value is not an array. + * + * @param jsonObject the object of arrays + * @return the names of each event key + */ + @Nonnull + private static Map> namesPerEventOf(@Nonnull JSONObject jsonObject) { + Map> namesPerEvent = new LinkedHashMap<>(); + for (String key : jsonObject.keySet()) { + JSONArray jsonArray = jsonObject.optJSONArray(key); + if (jsonArray != null) { + namesPerEvent.put(key, namesOf(jsonArray)); + } + } + return namesPerEvent; + } + + /** + * Decides the log gathering state from the top level {@code lg} directive, shaped + * {@code {"e":false}} or {@code {"e":true,"i":id,"l":"ewidv","b":100}}. Only a live response + * decides: without one the state stays as it is, while disabled settings requests, a response + * without a usable directive and a directive with {@code e} not {@code true} decide off. + * + * @param serverResponse the live response, {@code null} at init + */ + private synchronized void readLogGatheringDirective(@Nullable JSONObject serverResponse) { + if (serverConfigRequestsDisabled) { + //without settings requests no directive can ever arrive, so nothing can ever be gathered + L.d("[ModuleConfiguration] readLogGatheringDirective, SDK behavior settings requests are disabled, log gathering can never be armed"); + setLogGatheringOff(); + return; + } + + if (serverResponse == null) { + L.d("[ModuleConfiguration] readLogGatheringDirective, not a server response, log gathering stays undecided"); + return; + } + + JSONObject directive = serverResponse.optJSONObject(keyRLogGathering); + if (directive == null) { + L.d("[ModuleConfiguration] readLogGatheringDirective, server response had no usable '" + keyRLogGathering + "', log gathering is off"); + setLogGatheringOff(); + return; + } + + if (!Boolean.TRUE.equals(directive.opt(keyLGEnabled))) { + L.d("[ModuleConfiguration] readLogGatheringDirective, directive says log gathering is off"); + setLogGatheringOff(); + return; + } + + Object gatherIdRaw = directive.opt(keyLGId); + String gatherId = gatherIdRaw instanceof String ? ((String) gatherIdRaw).trim() : ""; + if (gatherId.isEmpty()) { + //without the id the server rejects every uploaded batch + L.d("[ModuleConfiguration] readLogGatheringDirective, directive enables log gathering but carries no usable '" + keyLGId + "', log gathering is off"); + setLogGatheringOff(); + return; + } + + LogGatheringDirective gathering = new LogGatheringDirective(LogGatheringState.GATHERING, gatherId, + sanitizeLogGatheringLevels(directive.opt(keyLGLevels)), sanitizeLogGatheringBatchSize(directive.opt(keyLGBatchSize))); + currentVLogGathering = gathering; + L.d("[ModuleConfiguration] readLogGatheringDirective, log gathering is on, id:[" + gathering.gatherId + "], levels:[" + gathering.levels + "], batch size:[" + gathering.batchSize + "]"); + } + + /** + * Decides against log gathering when nothing decided yet, for a run where no response can arrive, + * and has the logger apply the decision. + * + * @param reason why no response can arrive + */ + private synchronized void decideLogGatheringOffIfUndecided(@Nonnull String reason) { + if (currentVLogGathering.state == LogGatheringState.UNDECIDED) { + L.d("[ModuleConfiguration] decideLogGatheringOffIfUndecided, " + reason + ", log gathering is off"); + setLogGatheringOff(); + L.applyLogGatheringDirective(reason); + } + } + + /** + * Publishes the decision against log gathering, which also drops any previous gather id. + */ + private void setLogGatheringOff() { + currentVLogGathering = LogGatheringDirective.NOT_GATHERING; + } + + /** + * Stops log gathering before the device ID changes without merge: decides the directive off, + * then has the logger queue what it holds under the current device ID and stop, so nothing the + * next user does is gathered until a live response for the new device ID arms a gather. + */ + void stopLogGatheringForDeviceIdChange() { + synchronized (this) { + deviceIdChangesWithoutMerge++; + setLogGatheringOff(); + } + L.stopLogGatheringForDeviceIdChange(); + } + + /** + * Keeps only the level characters this SDK knows, lower cased, in the order the server sent them + * and without duplicates. Falls back to every level when nothing usable is left. + * + * @param levelsRaw the {@code l} value of the directive + * @return the level characters to gather + */ + @Nonnull + private String sanitizeLogGatheringLevels(@Nullable Object levelsRaw) { + if (!(levelsRaw instanceof String)) { + return logGatheringAllLevels; + } + + String levels = (String) levelsRaw; + StringBuilder filtered = new StringBuilder(); + for (int i = 0; i < levels.length(); i++) { + char level = Character.toLowerCase(levels.charAt(i)); + if (logGatheringAllLevels.indexOf(level) > -1 && filtered.indexOf(String.valueOf(level)) < 0) { + filtered.append(level); + } + } + + if (filtered.length() == 0) { + L.d("[ModuleConfiguration] sanitizeLogGatheringLevels, no usable level in [" + levels + "], falling back to [" + logGatheringAllLevels + "]"); + return logGatheringAllLevels; + } + + return filtered.toString(); + } + + /** + * Clamps the batch size into [{@value #logGatheringMinBatchSize}, {@value #logGatheringMaxBufferedLines}], + * falling back to {@value #logGatheringDefaultBatchSize} when it is missing or not a number. + * + * @param batchSizeRaw the {@code b} value of the directive + * @return the batch size + */ + private static int sanitizeLogGatheringBatchSize(@Nullable Object batchSizeRaw) { + if (!(batchSizeRaw instanceof Number)) { + return logGatheringDefaultBatchSize; + } + + int batchSize = ((Number) batchSizeRaw).intValue(); + return Math.min(logGatheringMaxBufferedLines, Math.max(logGatheringMinBatchSize, batchSize)); + } + + /** + * Requests the settings from the server and applies the response. Skipped when settings + * requests are disabled, in backend mode and while there is no device ID. The request is sent + * even while the settings forbid networking, as the next response is the only way to allow it again. + * + * @param config configuration of the running SDK + */ + void fetchConfigFromServer(@Nonnull InternalConfig config) { + L.v("[ModuleConfiguration] fetchConfigFromServer"); + if (!running || serverConfigRequestsDisabled) { + L.v("[ModuleConfiguration] fetchConfigFromServer, fetching the configuration is aborted, server config requests are disabled or the SDK is stopped"); + return; + } + + if (config.getDeviceId() == null) { + L.d("[ModuleConfiguration] fetchConfigFromServer, fetching the configuration is aborted, there is no device ID yet"); + decideLogGatheringOffIfUndecided("no device ID, no server response this run"); + return; + } + + if (config.sdk == null || config.sdk.networking == null) { + L.w("[ModuleConfiguration] fetchConfigFromServer, fetching the configuration is aborted, networking is not available"); + decideLogGatheringOffIfUndecided("networking is not available, no server response this run"); + return; + } + + try { + String requestData = ModuleRequests.prepareRequiredParams(config).add("method", "sc").toString(); + Transport transport = config.sdk.networking.getTransport(); + final long fetchStartNs = System.nanoTime(); + final int requestedAfterDeviceIdChanges = deviceIdChangesWithoutMerge; + + config.immediateRequestGenerator.createImmediateRequestMaker().doWork(requestData, "/o/sdk?", transport, false, true, + response -> onServerConfigResponse(config, response, fetchStartNs, requestedAfterDeviceIdChanges), L); + } catch (Exception e) { + L.e("[ModuleConfiguration] fetchConfigFromServer, failed to request the configuration, [" + e + "]"); + decideLogGatheringOffIfUndecided("the server config request failed"); + } + } + + /** + * Handles the response of a settings request: strips the connection test flag, stores and + * applies the settings, and tells the SDK when a resolved value changed. + * + * @param config configuration of the running SDK + * @param response the response, {@code null} when the request failed + * @param fetchStartNs {@link System#nanoTime()} just before the request was made + * @param requestedAfterDeviceIdChanges how many device ID changes without merge happened before the request was made + */ + private void onServerConfigResponse(@Nonnull InternalConfig config, @Nullable JSONObject response, long fetchStartNs, int requestedAfterDeviceIdChanges) { + if (!running) { + L.d("[ModuleConfiguration] onServerConfigResponse, the SDK was stopped before the response arrived, ignoring it"); + return; + } + + if (response == null) { + L.w("[ModuleConfiguration] onServerConfigResponse, not possible to retrieve configuration data, probably due to lack of connection to the server"); + decideLogGatheringOffIfUndecided("server config fetch failed"); + return; + } + + L.d("[ModuleConfiguration] onServerConfigResponse, retrieved configuration response: [" + response + "]"); + long fetchLatencyMs = (System.nanoTime() - fetchStartNs) / 1_000_000L; + boolean connectionTestRequested = extractConnectionTestFlag(response); + + boolean changed; + synchronized (this) { + //checked again under the lock stop() takes, so a response racing stop() never writes the store + if (!running) { + L.d("[ModuleConfiguration] onServerConfigResponse, the SDK was stopped while the response was read, ignoring it"); + return; + } + saveAndStoreDownloadedConfig(response); + changed = updateConfigVariables(config, response, requestedAfterDeviceIdChanges == deviceIdChangesWithoutMerge); + } + + if (changed && config.sdk != null) { + config.sdk.onSdkConfigurationChanged(config); + } + + if (connectionTestRequested) { + notifyConnectionTestRequested(fetchLatencyMs); + } + } + + /** + * Reads and removes the {@code ct} flag of a live response, so it is never stored and can never + * come back from storage. Booleans and numbers count by value; a string counts unless it is + * empty, {@code "0"} or {@code "false"}; any other value that is not null counts. + * + * @param serverConfigResponse the live response, changed in place + * @return whether the response asks for a connection test + */ + static boolean extractConnectionTestFlag(@Nullable JSONObject serverConfigResponse) { + if (serverConfigResponse == null || !serverConfigResponse.has(keyRConnectionTest)) { + return false; + } + + Object value = serverConfigResponse.remove(keyRConnectionTest); + if (value == null || JSONObject.NULL.equals(value)) { + return false; + } + if (value instanceof Boolean) { + return (Boolean) value; + } + if (value instanceof Number) { + return ((Number) value).doubleValue() != 0; + } + if (value instanceof String) { + String s = ((String) value).trim(); + return !s.isEmpty() && !"0".equals(s) && !"false".equalsIgnoreCase(s); + } + return true; + } + + /** + * Hands a requested connection test to the registered listener. + * + * @param fetchLatencyMs how long the settings request took, in milliseconds + */ + private void notifyConnectionTestRequested(long fetchLatencyMs) { + ConnectionTestListener listener = connectionTestListener; + if (listener == null) { + L.d("[ModuleConfiguration] notifyConnectionTestRequested, the server asked for a connection test, nothing is registered to run it"); + return; + } + + try { + listener.onConnectionTestRequested(fetchLatencyMs); + } catch (Exception e) { + L.e("[ModuleConfiguration] notifyConnectionTestRequested, the connection test listener failed, [" + e + "]"); + } + } + + /** + * Registers what runs the connection test a live settings response can ask for. + * + * @param listener called with the latency of the settings request, {@code null} to unregister + */ + void setConnectionTestListener(@Nullable ConnectionTestListener listener) { + connectionTestListener = listener; + } + + /** + * The refresh interval in the unit of the timer. + * + * @return seconds between two settings fetches + */ + private long serverConfigUpdateIntervalSeconds() { + return (long) currentVServerConfigUpdateInterval * 60L * 60L; + } + + /** + * Starts the timer that fetches the settings every {@code scui} hours, replacing a running one. + * Never waits for the replaced timer, as this also runs from that timer's own task. + */ + private synchronized void startServerConfigUpdateTimer() { + if (serverConfigUpdateTimer != null) { + serverConfigUpdateTimer.stopTimer(false); + } + + L.d("[ModuleConfiguration] startServerConfigUpdateTimer, fetching the configuration every [" + currentVServerConfigUpdateInterval + "] hours"); + final InternalConfig config = internalConfig; + serverConfigUpdateTimer = new CountlyTimer(L); + serverConfigUpdateTimer.startTimer(serverConfigUpdateIntervalSeconds(), () -> fetchConfigFromServer(config)); + } + + /** + * Stops the refresh timer, if it runs. + */ + private synchronized void stopServerConfigUpdateTimer() { + if (serverConfigUpdateTimer != null) { + serverConfigUpdateTimer.stopTimer(false); + serverConfigUpdateTimer = null; + } + } + + /** {@inheritDoc} */ + @Override + public boolean getNetworkingEnabled() { + return currentVNetworking; + } + + /** {@inheritDoc} */ + @Override + public boolean getTrackingEnabled() { + return currentVTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getSessionTrackingEnabled() { + return currentVSessionTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getViewTrackingEnabled() { + return currentVViewTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getCustomEventTrackingEnabled() { + return currentVCustomEventTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getContentZoneEnabled() { + return currentVContentZone; + } + + /** {@inheritDoc} */ + @Override + public boolean getCrashReportingEnabled() { + return currentVCrashReporting; + } + + /** {@inheritDoc} */ + @Override + public boolean getAutomaticSessionTrackingEnabled() { + return currentVAutomaticSessionTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getAutomaticViewTrackingEnabled() { + return currentVAutomaticViewTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getAutomaticCrashReportingEnabled() { + return currentVAutomaticCrashReporting; + } + + /** {@inheritDoc} */ + @Override + public boolean getLocationTrackingEnabled() { + return currentVLocationTracking; + } + + /** {@inheritDoc} */ + @Override + public boolean getRefreshContentZoneEnabled() { + return currentVRefreshContentZone; + } + + /** {@inheritDoc} */ + @Override + public boolean getBOMEnabled() { + return currentVBackoffMechanism; + } + + /** {@inheritDoc} */ + @Override + public boolean getLoggingEnabled() { + return currentVLoggingEnabled; + } + + /** {@inheritDoc} */ + @Override + public boolean getConsentRequired() { + return currentVRequiresConsent; + } + + /** {@inheritDoc} */ + @Override + public int getServerConfigUpdateInterval() { + return currentVServerConfigUpdateInterval; + } + + /** {@inheritDoc} */ + @Override + public int getRequestQueueMaxSize() { + return currentVRequestQueueMaxSize; + } + + /** {@inheritDoc} */ + @Override + public boolean isRequestQueueMaxSizeFromBehaviorSettings() { + return currentVRequestQueueMaxSizeFromBehaviorSettings; + } + + /** {@inheritDoc} */ + @Override + public int getEventQueueSizeThreshold() { + return currentVEventQueueSizeThreshold; + } + + /** {@inheritDoc} */ + @Override + public int getSessionUpdateInterval() { + return currentVSessionUpdateInterval; + } + + /** {@inheritDoc} */ + @Override + public int getMaxKeyLength() { + return currentVMaxKeyLength; + } + + /** {@inheritDoc} */ + @Override + public int getMaxValueSize() { + return currentVMaxValueSize; + } + + /** {@inheritDoc} */ + @Override + public int getMaxSegmentationValues() { + return currentVMaxSegmentationValues; + } + + /** {@inheritDoc} */ + @Override + public int getMaxBreadcrumbCount() { + return currentVMaxBreadcrumbCount; + } + + /** {@inheritDoc} */ + @Override + public int getMaxStackTraceLinesPerThread() { + return currentVMaxStackTraceLinesPerThread; + } + + /** {@inheritDoc} */ + @Override + public int getMaxStackTraceLineLength() { + return currentVMaxStackTraceLineLength; + } + + /** {@inheritDoc} */ + @Override + public int getUserPropertyCacheLimit() { + return currentVUserPropertyCacheLimit; + } + + /** {@inheritDoc} */ + @Override + public int getBOMAcceptedTimeoutSeconds() { + return currentVBOMAcceptedTimeoutSeconds; + } + + /** {@inheritDoc} */ + @Override + public double getBOMRQPercentage() { + return currentVBOMRQPercentage; + } + + /** {@inheritDoc} */ + @Override + public int getBOMRequestAge() { + return currentVBOMRequestAge; + } + + /** {@inheritDoc} */ + @Override + public int getBOMDuration() { + return currentVBOMDuration; + } + + /** {@inheritDoc} */ + @Override + public int getRequestDropAgeHours() { + return currentVDropAgeHours; + } + + /** {@inheritDoc} */ + @Override + public int getContentZoneTimerInterval() { + return currentVZoneTimerInterval; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public FilterList> getEventFilterList() { + return currentVEventFilterList; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public FilterList> getUserPropertyFilterList() { + return currentVUserPropertyFilterList; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public FilterList> getSegmentationFilterList() { + return currentVSegmentationFilterList; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public FilterList>> getEventSegmentationFilterList() { + return currentVEventSegmentationFilterList; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public Set getJourneyTriggerEvents() { + return currentVJourneyTriggerEvents; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public Set getJourneyTriggerViews() { + return currentVJourneyTriggerViews; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public LogGatheringState getLogGatheringState() { + return currentVLogGathering.state; + } + + /** {@inheritDoc} */ + @Nullable + @Override + public String getLogGatheringId() { + return currentVLogGathering.gatherId; + } + + /** {@inheritDoc} */ + @Nonnull + @Override + public String getLogGatheringLevels() { + return currentVLogGathering.levels; + } + + /** {@inheritDoc} */ + @Override + public int getLogGatheringBatchSize() { + return currentVLogGathering.batchSize; + } +} diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConnectionTest.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConnectionTest.java new file mode 100644 index 00000000..79b7370f --- /dev/null +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConnectionTest.java @@ -0,0 +1,603 @@ +package ly.count.sdk.java.internal; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Runs the connection test a live SDK behavior settings response asks for with its top level + * {@code ct} flag: one battery of parameterless GET probes against every endpoint the SDK depends + * on, graded row by row and queued as one {@code ct_results} report. The feature keys, their order + * and the {@code e} strings are the ones every Countly SDK reports, so the dashboard can group the + * results. + *

+ * The module is built for every SDK, whatever its features and consent, and listens to + * {@link ModuleConfiguration}, which reads the flag. One battery runs at a time, on a daemon thread + * of its own, and none while the settings switch networking off. Probes go straight to the server, + * past the request queue, so neither the backoff mechanism nor the request drop age can hold them + * back, while the report is an ordinary queued request. Nothing about the test is ever stored, so a + * response that asks again runs the battery again. + */ +class ModuleConnectionTest extends ModuleBase implements ModuleConfiguration.ConnectionTestListener { + static final String keyResults = "ct_results"; + static final String THREAD_NAME = "Countly-ConnectionTest"; + + static final int DEFAULT_PER_REQUEST_TIMEOUT_MS = 10_000; + static final long BATTERY_CAP_EXTRA_MS = 30_000; + static final int MAX_ROWS = 32; + static final int MAX_REPORT_BYTES = 8 * 1024; + static final int MAX_ERROR_LENGTH = 256; + + static final String errorTimeout = "timeout"; + static final String errorTransport = "transport error"; + static final String errorRedirected = "redirected"; + static final String errorNotRun = "not run: battery cap"; + //the qualifier of a success graded from an opaque browser probe: never produced here, but it always survives the size cap + static final String errorOpaque = "opaque"; + + /** + * The feature rows in report order. The row without paths is {@code sc}, measured from the + * settings request that asked for the test. + */ + static final ProbeRow[] ROWS = { + new ProbeRow("core", true, "/o/ping"), + new ProbeRow("core-write", false, "/i"), + new ProbeRow("sc", false), + new ProbeRow("rc", false, "/o/sdk?method=rc"), + new ProbeRow("ab", false, "/o/sdk?method=ab_fetch_variants"), + new ProbeRow("feedback", false, "/o/sdk?method=feedback"), + new ProbeRow("feedback-widget", false, "/o/surveys/nps/widget", "/o/surveys/survey/widget", "/o/feedback/widget"), + new ProbeRow("feedback-submit", false, "/i/feedback/inputs"), + new ProbeRow("content", false, "/o/sdk/content"), + new ProbeRow("feedback-page", false, "/feedback/nps", "/feedback/survey", "/feedback/rating"), + new ProbeRow("feedback-assets", true, "/surveys/images/ct-probe.png", "/star-rating/images/ct-probe.png"), + new ProbeRow("content-page", false, "/_external/content/"), + }; + + final AtomicBoolean batteryRunning = new AtomicBoolean(false); + volatile boolean halted = false; + //taken by stop() to set halted and by the battery to check it and queue the report, so a stopped SDK never gets one + private final Object reportLock = new Object(); + private volatile ModuleConfiguration listenedConfiguration = null; + + //package-private so tests can shrink the deadlines and take the network out + volatile int perRequestTimeoutMs = DEFAULT_PER_REQUEST_TIMEOUT_MS; + volatile long batteryCapExtraMs = BATTERY_CAP_EXTRA_MS; + volatile ProbeTransport transport = new HttpProbeTransport(); + + /** + * One feature row of the battery. + */ + static final class ProbeRow { + final String feature; + /** + * Whether only a 2xx counts as reached: {@code core} and {@code feedback-assets} have no + * legitimate 4xx answer, so anything else is a fault there. + */ + final boolean requiresSuccessStatus; + final String[] paths; + + /** + * Describes one row. + * + * @param feature the feature key, reported as {@code f} + * @param requiresSuccessStatus whether only a 2xx counts as reached + * @param paths the paths to probe, relative to the server URL; none for the {@code sc} row + */ + ProbeRow(@Nonnull String feature, boolean requiresSuccessStatus, @Nonnull String... paths) { + this.feature = feature; + this.requiresSuccessStatus = requiresSuccessStatus; + this.paths = paths.clone(); + } + } + + /** + * What one probe observed, before it is graded. + */ + static final class ProbeOutcome { + /** + * The HTTP status, {@code 0} when none could be read. + */ + int status; + /** + * From issuing the request to reading the whole response, or to the failure. + */ + long ms; + /** + * {@link #errorTimeout} or {@link #errorTransport} while the status is {@code 0}, otherwise {@code null}. + */ + String failure; + } + + /** + * One graded row of the report. + */ + static final class ResultRow { + final String feature; + boolean ok; + int status; + long ms; + String error; + int pathCount; + + /** + * Starts a row that reached nothing yet. + * + * @param feature the feature key + */ + ResultRow(@Nonnull String feature) { + this.feature = feature; + } + } + + /** + * Issues one probe. Tests replace it to take the network out. + */ + interface ProbeTransport { + /** + * Sends one bare GET and reads its status. + * + * @param url the probe URL + * @param timeoutMs the connect and the read timeout, in milliseconds + * @param transport the transport of the queued requests, whose pinning and custom request + * headers the probe reuses, {@code null} when the SDK runs without one + * @return what the probe observed + */ + @Nonnull + ProbeOutcome probe(@Nonnull String url, int timeoutMs, @Nullable Transport transport); + } + + /** + * Starts listening to the configuration module of the SDK for connection test requests. + * + * @param config configuration of the SDK being initialized + */ + @Override + public void init(InternalConfig config) { + super.init(config); + ModuleConfiguration configuration = config.sdk == null ? null : config.sdk.module(ModuleConfiguration.class); + if (configuration == null) { + L.d("[ModuleConnectionTest] init, no configuration module runs, a connection test can never be requested"); + return; + } + + configuration.setConnectionTestListener(this); + listenedConfiguration = configuration; + L.v("[ModuleConnectionTest] init, listening for connection test requests"); + } + + /** + * Stops listening for connection test requests. A running battery ends at its next row and + * never queues its report. + * + * @param config configuration of the SDK being stopped + * @param clear whether the SDK clears its data + */ + @Override + public void stop(InternalConfig config, boolean clear) { + super.stop(config, clear); + synchronized (reportLock) { + halted = true; + } + + ModuleConfiguration configuration = listenedConfiguration; + listenedConfiguration = null; + if (configuration != null) { + configuration.setConnectionTestListener(null); + } + + if (batteryRunning.get()) { + L.d("[ModuleConnectionTest] stop, the running battery ends at its next row and queues no report"); + } + } + + /** + * Runs the battery a live settings response asked for. + * + * @param fetchLatencyMs how long the settings request took, in milliseconds + */ + @Override + public void onConnectionTestRequested(long fetchLatencyMs) { + startBattery(fetchLatencyMs); + } + + /** + * Starts one battery on a daemon thread of its own, unless the SDK stopped, the settings switch + * networking off or a battery is already running. + * + * @param scLatencyMs latency of the settings request that asked for the test, in milliseconds, + * negative when unknown + */ + void startBattery(final long scLatencyMs) { + if (halted) { + L.d("[ModuleConnectionTest] startBattery, the SDK is stopped, ignoring the request"); + return; + } + + if (!internalConfig.getConfigurationProvider().getNetworkingEnabled()) { + L.d("[ModuleConnectionTest] startBattery, networking is disabled by the SDK behavior settings, ignoring the request"); + return; + } + + if (!batteryRunning.compareAndSet(false, true)) { + L.d("[ModuleConnectionTest] startBattery, a battery is already running, ignoring the request"); + return; + } + + L.i("[ModuleConnectionTest] startBattery, the server asked for a connection test, running the probe battery"); + Thread worker = new Thread(() -> { + try { + runBatteryAndReport(scLatencyMs); + } catch (Throwable t) { + L.e("[ModuleConnectionTest] startBattery, the battery failed, [" + t + "]"); + } finally { + batteryRunning.set(false); + } + }, THREAD_NAME); + worker.setDaemon(true); + + try { + worker.start(); + } catch (Throwable t) { + batteryRunning.set(false); + L.e("[ModuleConnectionTest] startBattery, failed to start the battery thread, [" + t + "]"); + } + } + + /** + * Runs the battery and queues its report as a request of its own, unless the SDK stopped + * meanwhile. + * + * @param scLatencyMs latency of the settings request that asked for the test, in milliseconds, + * negative when unknown + */ + void runBatteryAndReport(long scLatencyMs) { + List rows = runBattery(scLatencyMs); + String report = buildReport(rows); + + synchronized (reportLock) { + if (halted) { + L.d("[ModuleConnectionTest] runBatteryAndReport, the SDK stopped during the battery, dropping the report"); + return; + } + + L.d("[ModuleConnectionTest] runBatteryAndReport, queueing the report [" + report + "]"); + Request request = ModuleRequests.nonSessionRequest(internalConfig); + request.params.add(keyResults, report); + ModuleRequests.pushAsync(internalConfig, request); + } + } + + /** + * Probes the rows in report order, one request at a time. A row the battery cap is reached + * before is reported as not run, and a stop ends the battery before the next row. + * + * @param scLatencyMs latency of the settings request that asked for the test, in milliseconds, + * negative when unknown, which leaves the {@code sc} row out + * @return the graded rows in report order + */ + @Nonnull + List runBattery(long scLatencyMs) { + String serverUrl = internalConfig.getServerURL().toString(); + Transport sdkTransport = sdkTransport(); + ProbeTransport probeTransport = transport; + int timeoutMs = perRequestTimeoutMs; + + int requestCount = 0; + for (ProbeRow row : ROWS) { + requestCount += row.paths.length; + } + long capMs = (long) requestCount * timeoutMs + batteryCapExtraMs; + long batteryStartNs = System.nanoTime(); + + List results = new ArrayList<>(ROWS.length); + for (ProbeRow row : ROWS) { + if (halted) { + L.d("[ModuleConnectionTest] runBattery, the SDK stopped, ending the battery before [" + row.feature + "]"); + break; + } + + if (row.paths.length == 0) { + if (scLatencyMs >= 0) { + //the settings request was answered, or no test would have been asked for + ResultRow sc = new ResultRow(row.feature); + sc.ok = true; + sc.status = HttpURLConnection.HTTP_OK; + sc.ms = scLatencyMs; + results.add(sc); + } + continue; + } + + long elapsedMs = (System.nanoTime() - batteryStartNs) / 1_000_000L; + if (elapsedMs > capMs) { + ResultRow notRun = new ResultRow(row.feature); + notRun.error = errorNotRun; + results.add(notRun); + continue; + } + + results.add(probeRow(row, serverUrl, probeTransport, sdkTransport, timeoutMs)); + } + + L.d("[ModuleConnectionTest] runBattery, [" + results.size() + "] rows graded against [" + serverUrl + "]"); + return results; + } + + /** + * Probes every path of a row and combines them into one row: reached only when every path was, + * the status and the reason of the first failing path, or the first status when none failed, the + * latencies summed and the number of paths. + * + * @param row the row + * @param serverUrl the configured server URL + * @param probeTransport what issues the probes + * @param sdkTransport the transport of the queued requests, {@code null} when there is none + * @param timeoutMs the per request timeout, in milliseconds + * @return the graded row + */ + @Nonnull + ResultRow probeRow(@Nonnull ProbeRow row, @Nonnull String serverUrl, @Nonnull ProbeTransport probeTransport, @Nullable Transport sdkTransport, int timeoutMs) { + ResultRow result = new ResultRow(row.feature); + result.ok = true; + result.pathCount = row.paths.length; + + for (int i = 0; i < row.paths.length; i++) { + String url = buildProbeUrl(serverUrl, row.paths[i]); + ProbeOutcome outcome = probeTransport.probe(url, timeoutMs, sdkTransport); + result.ms += outcome.ms; + + String failure = grade(outcome, row.requiresSuccessStatus); + L.v("[ModuleConnectionTest] probeRow, [" + row.feature + "] " + url + " -> status:[" + outcome.status + "] ms:[" + outcome.ms + "] failure:[" + failure + "]"); + + if (i == 0) { + result.status = outcome.status; + } + + if (failure != null && result.ok) { + result.ok = false; + result.status = outcome.status; + result.error = failure; + } + } + + return result; + } + + /** + * Grades one probe by whether the Countly application answered it: a 2xx did, and so did a + * 4xx other than 403 on a row that has a legitimate 4xx answer. No status, a redirect, a 403, a + * 5xx and anything else did not. + * + * @param outcome what the probe observed + * @param requiresSuccessStatus whether only a 2xx counts as reached on this row + * @return {@code null} when the application answered, otherwise the {@code e} reason + */ + @Nullable + static String grade(@Nonnull ProbeOutcome outcome, boolean requiresSuccessStatus) { + int status = outcome.status; + if (status <= 0) { + return outcome.failure != null ? outcome.failure : errorTransport; + } + if (status >= 200 && status < 300) { + return null; + } + if (status >= 300 && status < 400) { + return errorRedirected; + } + if (!requiresSuccessStatus && status >= 400 && status < 500 && status != HttpURLConnection.HTTP_FORBIDDEN) { + return null; + } + return "HTTP " + status; + } + + /** + * Resolves a path against the configured server URL, keeping any path prefix, with exactly one + * {@code /} at the join, then appends the probe marker and a cache buster. + * + * @param serverUrl the configured server URL + * @param path the path, which may carry a query + * @return the probe URL + */ + @Nonnull + static String buildProbeUrl(@Nonnull String serverUrl, @Nonnull String path) { + String base = serverUrl; + while (base.endsWith("/")) { + base = base.substring(0, base.length() - 1); + } + String joined = path.startsWith("/") ? base + path : base + "/" + path; + String separator = joined.contains("?") ? "&" : "?"; + return joined + separator + "ct=1&_=" + System.currentTimeMillis(); + } + + /** + * Builds the report: the device time, the SDK identity and the rows, within the caps of + * {@value #MAX_ROWS} rows and {@value #MAX_REPORT_BYTES} bytes of UTF-8. Over the size cap every + * {@code e} except {@link #errorOpaque} is dropped first, then rows from the end. + * + * @param rows the graded rows in report order + * @return the report as JSON text + */ + @Nonnull + String buildReport(@Nonnull List rows) { + JSONObject report = new JSONObject(); + JSONArray results = new JSONArray(); + try { + report.put("ts", System.currentTimeMillis()); + JSONObject sdk = new JSONObject(); + sdk.put("name", internalConfig.getSdkName()); + sdk.put("version", internalConfig.getSdkVersion()); + report.put("sdk", sdk); + report.put("results", results); + + int rowLimit = Math.min(rows.size(), MAX_ROWS); + for (int i = 0; i < rowLimit; i++) { + results.put(rowToJson(rows.get(i))); + } + + if (utf8Length(report.toString()) > MAX_REPORT_BYTES) { + L.w("[ModuleConnectionTest] buildReport, the report is over the size cap, dropping the error details"); + for (int i = 0; i < results.length(); i++) { + JSONObject row = results.getJSONObject(i); + if (!errorOpaque.equals(row.optString("e"))) { + row.remove("e"); + } + } + } + + while (utf8Length(report.toString()) > MAX_REPORT_BYTES && results.length() > 0) { + L.w("[ModuleConnectionTest] buildReport, the report is still over the size cap, dropping its last row"); + results.remove(results.length() - 1); + } + } catch (JSONException e) { + L.w("[ModuleConnectionTest] buildReport, failed to build the report, [" + e + "]"); + } + + return report.toString(); + } + + /** + * Serializes one row, its {@code e} cut at {@value #MAX_ERROR_LENGTH} characters and its path + * count only when it probed more than one path. + * + * @param row the graded row + * @return the row as the report carries it + */ + @Nonnull + static JSONObject rowToJson(@Nonnull ResultRow row) { + JSONObject json = new JSONObject(); + json.put("f", row.feature); + json.put("ok", row.ok); + json.put("st", row.status); + json.put("ms", row.ms); + if (row.error != null) { + String error = row.error; + if (error.length() > MAX_ERROR_LENGTH) { + error = error.substring(0, MAX_ERROR_LENGTH); + } + json.put("e", error); + } + if (row.pathCount > 1) { + json.put("n", row.pathCount); + } + return json; + } + + /** + * The size of a text in UTF-8. + * + * @param value the text + * @return its length in bytes + */ + static int utf8Length(@Nonnull String value) { + return value.getBytes(StandardCharsets.UTF_8).length; + } + + /** + * The transport the queued requests go through, whose pinning and custom request headers the + * probes reuse. + * + * @return the transport, {@code null} when the SDK runs without its default networking + */ + @Nullable + private Transport sdkTransport() { + SDKCore sdk = internalConfig.sdk; + Networking networking = sdk == null ? null : sdk.networking; + return networking == null ? null : networking.getTransport(); + } + + /** + * Probes with a bare GET opened by {@link Transport#openProbeConnection(String, int)}. + */ + static final class HttpProbeTransport implements ProbeTransport { + //only the status matters, but the body is read so 'ms' covers the whole response, up to this much of it + static final int MAX_DRAIN_BYTES = 64 * 1024; + + /** + * Sends the probe and reads its status and its body, mapping a timeout to + * {@link #errorTimeout} and any other failure to {@link #errorTransport}. + * + * @param url the probe URL + * @param timeoutMs the connect and the read timeout, in milliseconds + * @param transport the transport whose pinning and custom request headers the probe reuses, + * {@code null} for a plain connection + * @return what the probe observed + */ + @Nonnull + @Override + public ProbeOutcome probe(@Nonnull String url, int timeoutMs, @Nullable Transport transport) { + ProbeOutcome outcome = new ProbeOutcome(); + long startNs = System.nanoTime(); + HttpURLConnection connection = null; + try { + connection = transport == null ? Transport.openBareGetConnection(url, timeoutMs) : transport.openProbeConnection(url, timeoutMs); + int status = connection.getResponseCode(); + if (status > 0) { + outcome.status = status; + drain(connection); + } else { + //not an HTTP answer + outcome.failure = errorTransport; + } + } catch (SocketTimeoutException e) { + outcome.status = 0; + outcome.failure = errorTimeout; + } catch (Exception e) { + outcome.status = 0; + outcome.failure = errorTransport; + } finally { + if (connection != null) { + connection.disconnect(); + } + outcome.ms = (System.nanoTime() - startNs) / 1_000_000L; + } + return outcome; + } + + /** + * Reads the body, up to {@link #MAX_DRAIN_BYTES}, and discards it. + * + * @param connection the answered connection + */ + private static void drain(@Nonnull HttpURLConnection connection) { + try (InputStream stream = bodyStream(connection)) { + if (stream == null) { + return; + } + byte[] buffer = new byte[4096]; + int total = 0; + int read; + while (total < MAX_DRAIN_BYTES && (read = stream.read(buffer)) != -1) { + total += read; + } + } catch (IOException ignored) { + //the status is already read, a failed body read changes nothing + } + } + + /** + * The body of the answer, which is the error stream for a failing status. + * + * @param connection the answered connection + * @return the body, {@code null} when there is none + */ + @Nullable + private static InputStream bodyStream(@Nonnull HttpURLConnection connection) { + try { + return connection.getInputStream(); + } catch (IOException e) { + return connection.getErrorStream(); + } + } + } +} diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleContent.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleContent.java index c60d04d9..90721f7b 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleContent.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleContent.java @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import ly.count.sdk.java.Countly; import org.json.JSONArray; @@ -49,8 +50,21 @@ public class ModuleContent extends ModuleBase { private boolean fetching = false; private int waitForDelay = 0; private int generation = 0; - - private int zoneTimerInterval = ConfigContent.DEFAULT_ZONE_TIMER_INTERVAL; + /** + * Whether the active zone was entered because the {@code ecz} setting asked for it, so turning + * that setting off leaves it. Guarded by {@link #contentLock}. + */ + private boolean zoneEnteredBySettings = false; + /** + * The {@code ecz} value this module last acted on, so a settings change acts only when that + * value changed. Guarded by {@link #contentLock}. + */ + private boolean contentZoneEnabledApplied = false; + /** + * Set once {@link #stop(InternalConfig, boolean)} starts, after which no zone is entered, so no + * zone timer outlives the module. Guarded by {@link #contentLock}. + */ + private boolean stopped = false; /** When the SDK came up, which is what {@link #START_DELAY_MS} is measured from. */ private long startedAt = 0; @@ -64,10 +78,103 @@ public void init(InternalConfig config) { super.init(config); L.v("[ModuleContent] Initializing"); - zoneTimerInterval = config.content.zoneTimerInterval; globalContentCallback = config.content.globalContentCallback; contentInterface = new Content(); startedAt = TimeUtils.uniqueTimestampMs(); + synchronized (contentLock) { + display = config.sdk.contentDisplay; + contentZoneEnabledApplied = config.getConfigurationProvider().getContentZoneEnabled(); + } + } + + /** + * Enters the content zone when the {@code ecz} setting asks for it. While no display is + * registered the zone is entered once one is. + * + * @param config configuration of the running SDK + */ + @Override + protected void initFinished(InternalConfig config) { + super.initFinished(config); + if (config.getConfigurationProvider().getContentZoneEnabled()) { + enterContentZoneForSettings("initFinished"); + } + } + + /** + * Applies the content settings a server response changed. A changed {@code czi} restarts the + * timer of an active zone. A changed {@code ecz} enters the zone when it turned on, and when it + * turned off leaves the zone only if this setting entered it: a zone entered through + * {@link Content#enterContentZone()} is left to the application. + * + * @param config configuration of the running SDK + */ + @Override + protected void onSdkConfigurationChanged(InternalConfig config) { + ConfigurationProvider configProvider = config.getConfigurationProvider(); + boolean contentZoneEnabled = configProvider.getContentZoneEnabled(); + boolean contentZoneSettingChanged; + synchronized (contentLock) { + contentZoneSettingChanged = contentZoneEnabled != contentZoneEnabledApplied; + contentZoneEnabledApplied = contentZoneEnabled; + } + + restartZoneTimerIfIntervalChanged(configProvider.getContentZoneTimerInterval()); + + if (!contentZoneSettingChanged) { + return; + } + + if (contentZoneEnabled) { + enterContentZoneForSettings("onSdkConfigurationChanged"); + } else if (exitContentZoneInternal(true, true)) { + L.i("[ModuleContent] onSdkConfigurationChanged, the SDK behavior settings turned the content zone off, left the zone they had entered"); + } else { + L.d("[ModuleContent] onSdkConfigurationChanged, the SDK behavior settings turned the content zone off, no zone they entered is active"); + } + } + + /** + * Enters the content zone because the {@code ecz} setting asks for it, once a display is registered. + * + * @param caller the calling method, for the log + */ + private void enterContentZoneForSettings(@Nonnull String caller) { + boolean hasDisplay; + synchronized (contentLock) { + hasDisplay = display != null; + } + + if (!hasDisplay) { + L.d("[ModuleContent] " + caller + ", the SDK behavior settings ask for the content zone, it is entered once a content display is registered"); + return; + } + + L.d("[ModuleContent] " + caller + ", entering the content zone as the SDK behavior settings ask"); + enterContentZoneInternal(true); + } + + /** + * Restarts the timer of an active zone when the fetch interval changed, keeping the zone as it + * is. The next fetch comes as soon as the start delay allows. + * + * @param intervalSeconds the fetch interval now in effect + */ + private void restartZoneTimerIfIntervalChanged(int intervalSeconds) { + CountlyTimer timerToStop; + synchronized (contentLock) { + if (!zoneActive || contentTimer == null || contentTimer.getTimerDelaySeconds() == intervalSeconds) { + return; + } + + L.d("[ModuleContent] restartZoneTimerIfIntervalChanged, the fetch interval changed from [" + contentTimer.getTimerDelaySeconds() + "] to [" + intervalSeconds + "] seconds, restarting the zone timer"); + timerToStop = contentTimer; + contentTimer = new CountlyTimer(L); + contentTimer.startTimer(intervalSeconds, firstFetchDelay(), this::onZoneTimerTick); + } + + // Not awaited: the zone stays active, so a tick of the replaced timer that is still running is harmless. + timerToStop.stopTimer(false); } @Override @@ -75,8 +182,18 @@ public Boolean onRequest(Request request) { return true; } + /** + * Leaves the content zone for good: from here on no zone is entered, not even by a settings + * change that arrives while the SDK stops. + * + * @param config configuration of the SDK being stopped + * @param clear whether the SDK clears its data + */ @Override public void stop(InternalConfig config, boolean clear) { + synchronized (contentLock) { + stopped = true; + } super.stop(config, clear); exitContentZoneInternal(); display = null; @@ -105,12 +222,30 @@ protected void deviceIdChanged(String oldDeviceId, boolean withMerge) { void setContentDisplayInternal(ContentDisplay contentDisplay) { L.d("[ModuleContent] setContentDisplayInternal, display set:[" + (contentDisplay != null) + "]"); + boolean zoneInactive; synchronized (contentLock) { display = contentDisplay; + zoneInactive = !zoneActive; + } + + if (contentDisplay != null && zoneInactive && internalConfig.getConfigurationProvider().getContentZoneEnabled()) { + enterContentZoneForSettings("setContentDisplayInternal"); } } void enterContentZoneInternal() { + enterContentZoneInternal(false); + } + + /** + * Enters the content zone, fetching every {@code czi} seconds. Entering a zone that is already + * active does nothing, except that a call made for the application keeps a zone the settings + * entered active until the application leaves it. + * + * @param bySettings {@code true} when the {@code ecz} setting asks for the zone, {@code false} + * when the application does + */ + private void enterContentZoneInternal(boolean bySettings) { if (display == null) { L.w("[ModuleContent] enterContentZoneInternal, no content display is registered, ignoring the call"); return; @@ -121,9 +256,20 @@ void enterContentZoneInternal() { return; } + int zoneTimerInterval = internalConfig.getConfigurationProvider().getContentZoneTimerInterval(); synchronized (contentLock) { + if (stopped) { + L.d("[ModuleContent] enterContentZoneInternal, the module is stopped, ignoring the call"); + return; + } + if (zoneActive) { - L.d("[ModuleContent] enterContentZoneInternal, already in a content zone, ignoring the call"); + if (!bySettings && zoneEnteredBySettings) { + zoneEnteredBySettings = false; + L.d("[ModuleContent] enterContentZoneInternal, already in the content zone the SDK behavior settings entered, it now stays until it is left through exitContentZone"); + } else { + L.d("[ModuleContent] enterContentZoneInternal, already in a content zone, ignoring the call"); + } return; } @@ -132,6 +278,7 @@ void enterContentZoneInternal() { contentShown = false; fetching = false; waitForDelay = 0; + zoneEnteredBySettings = bySettings; // Any fetch left in flight from a previous zone belongs to an older generation and is // discarded when it completes. generation++; @@ -172,13 +319,31 @@ void exitContentZoneInternal() { * tick, because a task cannot wait for itself to finish */ private void exitContentZoneInternal(boolean awaitTimerTermination) { + exitContentZoneInternal(awaitTimerTermination, false); + } + + /** + * Leaves the content zone. + * + * @param awaitTimerTermination must be {@code false} when called from the zone timer's own + * tick, because a task cannot wait for itself to finish + * @param onlyIfEnteredBySettings {@code true} to leave only an active zone the {@code ecz} + * setting entered + * @return whether the zone was left + */ + private boolean exitContentZoneInternal(boolean awaitTimerTermination, boolean onlyIfEnteredBySettings) { CountlyTimer timerToStop; synchronized (contentLock) { + if (onlyIfEnteredBySettings && !(zoneActive && zoneEnteredBySettings)) { + return false; + } + zoneActive = false; shouldFetch = false; contentShown = false; fetching = false; waitForDelay = 0; + zoneEnteredBySettings = false; generation++; timerToStop = contentTimer; @@ -191,14 +356,27 @@ private void exitContentZoneInternal(boolean awaitTimerTermination) { } L.i("[ModuleContent] exitContentZoneInternal, left the content zone"); + return true; } + /** + * Flushes the event queue and enters the content zone again, keeping who entered it. Called by + * the application and when the server accepted a journey trigger; ignored while the + * {@code rcz} setting forbids refreshing and while a content block is on screen. + */ void refreshContentZoneInternal() { + if (!internalConfig.getConfigurationProvider().getRefreshContentZoneEnabled()) { + L.d("[ModuleContent] refreshContentZoneInternal, refreshing the content zone is disabled by the SDK behavior settings, ignoring the call"); + return; + } + + boolean enteredBySettings; synchronized (contentLock) { if (contentShown) { L.d("[ModuleContent] refreshContentZoneInternal, a content block is on screen, ignoring the call"); return; } + enteredBySettings = zoneActive && zoneEnteredBySettings; } // Push whatever is queued out first, so the trigger the developer just recorded has a @@ -206,7 +384,7 @@ void refreshContentZoneInternal() { flushEventQueue(); exitContentZoneInternal(); - enterContentZoneInternal(); + enterContentZoneInternal(enteredBySettings); } void previewContentInternal(String contentId) { @@ -240,11 +418,13 @@ void recordContentEventsInternal(String eventsJson) { return; } - ModuleEvents.Events events = SDKCore.instance == null ? null : SDKCore.instance.events(); - if (events == null) { + SDKCore core = SDKCore.instance; + ModuleEvents eventsModule = core == null ? null : core.runningModule(CoreFeature.Events, ModuleEvents.class); + if (eventsModule == null) { L.w("[ModuleContent] recordContentEventsInternal, events are not available, content events are dropped"); return; } + ModuleEvents.Events events = eventsModule.eventsInterface; boolean recorded = false; try { @@ -466,25 +646,57 @@ private void clearFetching(int fetchGeneration) { */ public class Content { + /** + * The interface a call goes to: this one while its module runs, the running module's when + * this one was handed out before that started or kept from one that stopped, and + * {@code null} while the feature does not run. + * + * @param call the call, for the log line + * @return the interface, {@code null} for none + */ + private Content target(String call) { + ModuleContent running = runningInstance(call); + return running == null ? null : running.contentInterface; + } + /** * Register the display that draws content blocks. Required before entering a content zone. - * Pass {@code null} to unregister. + * Pass {@code null} to unregister. While the SDK behavior settings ask for the content zone, + * registering a display enters it. A display registered before the content consent is given + * is used once it is. * * @param contentDisplay the display to draw content with * @apiNote This is an EXPERIMENTAL feature, and it can have breaking changes */ public void setContentDisplay(@Nullable ContentDisplay contentDisplay) { synchronized (Countly.instance()) { - setContentDisplayInternal(contentDisplay); + SDKCore core = SDKCore.instance; + if (core == null) { + return; + } + core.contentDisplay = contentDisplay; + ModuleContent running = core.module(ModuleContent.class); + if (running != null) { + running.setContentDisplayInternal(contentDisplay); + } } } /** * Start asking the server for content to show. Ignored while already in a content zone. + * Once this is called, the zone stays even when the SDK behavior settings turn the content + * zone off. * * @apiNote This is an EXPERIMENTAL feature, and it can have breaking changes */ public void enterContentZone() { + Content target = target("[Content] enterContentZone"); + if (target != this) { + if (target != null) { + target.enterContentZone(); + } + return; + } synchronized (Countly.instance()) { L.i("[Content] enterContentZone, entering the content zone"); enterContentZoneInternal(); @@ -498,6 +710,13 @@ public void enterContentZone() { * @apiNote This is an EXPERIMENTAL feature, and it can have breaking changes */ public void exitContentZone() { + Content target = target("[Content] exitContentZone"); + if (target != this) { + if (target != null) { + target.exitContentZone(); + } + return; + } synchronized (Countly.instance()) { L.i("[Content] exitContentZone, leaving the content zone"); exitContentZoneInternal(); @@ -506,11 +725,19 @@ public void exitContentZone() { /** * Re-enter the content zone right away, after flushing the event queue. Use it when a - * trigger condition just changed. Ignored while a content block is on screen. + * trigger condition just changed. Ignored while a content block is on screen and while the + * SDK behavior settings do not allow refreshing the content zone. * * @apiNote This is an EXPERIMENTAL feature, and it can have breaking changes */ public void refreshContentZone() { + Content target = target("[Content] refreshContentZone"); + if (target != this) { + if (target != null) { + target.refreshContentZone(); + } + return; + } synchronized (Countly.instance()) { L.i("[Content] refreshContentZone, refreshing the content zone"); refreshContentZoneInternal(); @@ -525,6 +752,13 @@ public void refreshContentZone() { * @apiNote This is an EXPERIMENTAL feature, and it can have breaking changes */ public void previewContent(@Nullable String contentId) { + Content target = target("[Content] previewContent"); + if (target != this) { + if (target != null) { + target.previewContent(contentId); + } + return; + } synchronized (Countly.instance()) { L.i("[Content] previewContent, previewing content:[" + contentId + "]"); if (Utils.isEmptyOrNull(contentId)) { @@ -544,6 +778,13 @@ public void previewContent(@Nullable String contentId) { * @apiNote This is an EXPERIMENTAL feature, and it can have breaking changes */ public void recordContentEvents(@Nullable String eventsJson) { + Content target = target("[Content] recordContentEvents"); + if (target != this) { + if (target != null) { + target.recordContentEvents(eventsJson); + } + return; + } synchronized (Countly.instance()) { L.d("[Content] recordContentEvents, recording the events of a content block"); recordContentEventsInternal(eventsJson); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleCrashes.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleCrashes.java index 2dbfb15a..e2d07128 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleCrashes.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleCrashes.java @@ -3,6 +3,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import ly.count.sdk.java.Countly; import ly.count.sdk.java.Crash; import ly.count.sdk.java.CrashProcessor; @@ -17,9 +19,16 @@ public class ModuleCrashes extends ModuleBase { private boolean crashed = false; protected InternalConfig config; + private final Object crashHandlerLock = new Object(); + //guarded by crashHandlerLock private Thread.UncaughtExceptionHandler previousHandler = null; + //guarded by crashHandlerLock + private Thread.UncaughtExceptionHandler installedHandler = null; + //written under crashHandlerLock + private volatile boolean stopped = false; protected CrashProcessor crashProcessor = null; - protected List logs = new ArrayList<>(); + //guarded by itself, as breadcrumbs are added on any thread and taken on the crash handler thread + protected final List logs = new ArrayList<>(); Crashes crashInterface; @@ -41,47 +50,142 @@ public void init(InternalConfig config) { @Override public void stop(InternalConfig config, boolean clear) { try { - if (previousHandler != null) { - Thread.setDefaultUncaughtExceptionHandler(previousHandler); - } + restoreUncaughtExceptionHandler(); if (clear) { config.sdk.sdkStorage.storablePurge(config, CrashImpl.getStoragePrefix()); } } catch (Throwable t) { L.e("[ModuleCrash] Exception while stopping crash reporting" + t); } - logs.clear(); + synchronized (logs) { + logs.clear(); + } crashInterface = null; } @Override public void initFinished(final InternalConfig config) { - previousHandler = Thread.getDefaultUncaughtExceptionHandler(); + installUncaughtExceptionHandlerIfEnabled("initFinished"); + started = System.nanoTime(); + } + + /** + * Installs the uncaught exception handler when a server response enabled automatic crash + * reporting and it is not installed yet. + * + * @param config configuration of the running SDK + */ + @Override + protected void onSdkConfigurationChanged(InternalConfig config) { + installUncaughtExceptionHandlerIfEnabled("onSdkConfigurationChanged"); + } - if (internalConfig.sdk.hasConsentForFeature(CoreFeature.CrashReporting) && config.isUnhandledCrashReportingEnabled()) { - registerUncaughtExceptionHandler(); + /** + * Installs the uncaught exception handler once, while crash reporting has consent and the SDK + * behavior settings enable both crash reporting and automatic crash reporting. An installed + * handler stays until the module stops and checks both settings again for every crash. + * + * @param caller the calling function, for the log + */ + private void installUncaughtExceptionHandlerIfEnabled(@Nonnull String caller) { + if (stopped) { + return; } - started = System.nanoTime(); + if (!isAutomaticCrashReportingEnabled()) { + L.d("[ModuleCrash] " + caller + ", automatic crash reporting is disabled, the uncaught exception handler is not installed"); + return; + } + + if (!internalConfig.sdk.hasConsentForFeature(CoreFeature.CrashReporting)) { + L.d("[ModuleCrash] " + caller + ", crash reporting has no consent, the uncaught exception handler is not installed"); + return; + } + + boolean installed = false; + synchronized (crashHandlerLock) { + if (installedHandler == null && !stopped) { + previousHandler = Thread.getDefaultUncaughtExceptionHandler(); + installedHandler = createUncaughtExceptionHandler(previousHandler); + Thread.setDefaultUncaughtExceptionHandler(installedHandler); + installed = true; + } + } + + if (installed) { + L.d("[ModuleCrash] " + caller + ", installed the uncaught exception handler"); + } } - private void registerUncaughtExceptionHandler() { - final Thread.UncaughtExceptionHandler handler = Thread.getDefaultUncaughtExceptionHandler(); - Thread.setDefaultUncaughtExceptionHandler((thread, throwable) -> { + /** + * Whether the SDK behavior settings enable both crash reporting and automatic crash reporting. + * + * @return {@code true} when unhandled crashes may be recorded + */ + private boolean isAutomaticCrashReportingEnabled() { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + return configProvider.getCrashReportingEnabled() && configProvider.getAutomaticCrashReportingEnabled(); + } + + /** + * Builds the handler that records an unhandled crash, while the module runs and the SDK behavior + * settings enable automatic crash reporting, and then always hands the crash to the handler it replaced. + * + * @param previous the handler to hand every crash to, {@code null} for none + * @return the handler + */ + @Nonnull + private Thread.UncaughtExceptionHandler createUncaughtExceptionHandler(@Nullable final Thread.UncaughtExceptionHandler previous) { + return (thread, throwable) -> { // needed since following UncaughtExceptionHandler can keep reference to this one crashed = true; - if (isActive()) { - recordExceptionInternal(throwable, false, null, null); + try { + if (isActive() && !stopped && isAutomaticCrashReportingEnabled()) { + recordExceptionInternal(throwable, false, null, null); + } + } catch (Throwable recordFailure) { + L.e("[ModuleCrash] uncaughtException, failed to record the crash, [" + recordFailure + "]"); + } finally { + if (previous != null) { + previous.uncaughtException(thread, throwable); + } } + }; + } - if (handler != null) { - handler.uncaughtException(thread, throwable); + /** + * Stops the installed handler from recording and puts back the handler it replaced, if it is + * still the default one. A handler installed over it keeps it in the chain, where it only hands + * crashes on. + */ + private void restoreUncaughtExceptionHandler() { + String outcome = null; + synchronized (crashHandlerLock) { + stopped = true; + if (installedHandler != null) { + if (Thread.getDefaultUncaughtExceptionHandler() == installedHandler) { + Thread.setDefaultUncaughtExceptionHandler(previousHandler); + outcome = "restored the uncaught exception handler it replaced"; + } else { + outcome = "another uncaught exception handler was installed over this one, which stays in the chain and only hands crashes on"; + } + installedHandler = null; + previousHandler = null; } - }); + } + + if (outcome != null) { + L.d("[ModuleCrash] stop, " + outcome); + } } protected void recordExceptionInternal(Throwable t, boolean handled, Map segments, String legacyCrashName) { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + if (!configProvider.getTrackingEnabled() || !configProvider.getCrashReportingEnabled()) { + L.d("[ModuleCrash] recordExceptionInternal, crash reporting disabled by SDK behavior settings; ignoring"); + return; + } if (config.isBackendModeEnabled()) { L.w("[ModuleCrash] recordExceptionInternal, Skipping crash, backend mode is enabled!"); return; @@ -114,9 +218,11 @@ public CrashImpl onCrash(InternalConfig config, CrashImpl crash) { L.w("[ModuleCrash] onCrash, While recording an exception 'App version' was either null or empty"); } - if (!logs.isEmpty()) { - crash.setLogs(logs.toArray(new String[0])); - logs.clear(); + synchronized (logs) { + if (!logs.isEmpty()) { + crash.setLogs(logs.toArray(new String[0])); + logs.clear(); + } } L.i("[ModuleCrash] onCrash: " + crash.getJSON()); @@ -134,6 +240,9 @@ public CrashImpl onCrash(InternalConfig config, CrashImpl crash) { L.e("[ModuleCrash] Error when calling CrashProcessor#process(Crash)" + t); } } + + //after the processor, so it matches against the whole crash and what it adds is limited too + crash.applyInternalLimits(config.getConfigurationProvider(), "[ModuleCrash] onCrash"); if (!Storage.push(config, crash)) { L.e("[ModuleCrash] Couldn't persist a crash, so dumping it here: " + crash.getJSON()); } else { @@ -148,21 +257,48 @@ protected void addBreadcrumbInternal(String record) { return; } - if (logs.size() >= config.getMaxBreadcrumbCount()) { - logs.remove(0); - } + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + String breadcrumb = UtilsInternalLimits.truncateValue(record, configProvider.getMaxValueSize(), L, "[ModuleCrash] addBreadcrumbInternal"); + + int maxBreadcrumbCount = configProvider.getMaxBreadcrumbCount(); + synchronized (logs) { + //a loop, as the limit can shrink while breadcrumbs are kept + while (!logs.isEmpty() && logs.size() >= maxBreadcrumbCount) { + logs.remove(0); + } - logs.add(record); + logs.add(breadcrumb); + } } public class Crashes { + /** + * The interface a call goes to: this one while its module runs, the running module's when + * this one was handed out before that started or kept from one that stopped, and + * {@code null} while the feature does not run. + * + * @param call the call, for the log line + * @return the interface, {@code null} for none + */ + private Crashes target(String call) { + ModuleCrashes running = runningInstance(call); + return running == null ? null : running.crashInterface; + } + /** * Add crash breadcrumb like log record to the log that will be sent together with crash report * * @param record String a bread crumb for the crash report */ public void addCrashBreadcrumb(String record) { + Crashes target = target("[Crashes] addCrashBreadcrumb"); + if (target != this) { + if (target != null) { + target.addCrashBreadcrumb(record); + } + return; + } synchronized (Countly.instance()) { L.i("[Crashes] Adding crash breadcrumb"); addBreadcrumbInternal(record); @@ -175,6 +311,13 @@ public void addCrashBreadcrumb(String record) { * @param exception Throwable to log */ public void recordHandledException(Throwable exception) { + Crashes target = target("[Crashes] recordHandledException"); + if (target != this) { + if (target != null) { + target.recordHandledException(exception); + } + return; + } synchronized (Countly.instance()) { recordExceptionInternal(exception, true, null, null); } @@ -186,6 +329,13 @@ public void recordHandledException(Throwable exception) { * @param exception Throwable to log */ public void recordUnhandledException(Throwable exception) { + Crashes target = target("[Crashes] recordUnhandledException"); + if (target != this) { + if (target != null) { + target.recordUnhandledException(exception); + } + return; + } synchronized (Countly.instance()) { recordExceptionInternal(exception, false, null, null); } @@ -197,6 +347,13 @@ public void recordUnhandledException(Throwable exception) { * @param exception Throwable to log */ public void recordHandledException(final Throwable exception, final Map customSegmentation) { + Crashes target = target("[Crashes] recordHandledException"); + if (target != this) { + if (target != null) { + target.recordHandledException(exception, customSegmentation); + } + return; + } synchronized (Countly.instance()) { recordExceptionInternal(exception, true, customSegmentation, null); } @@ -208,6 +365,13 @@ public void recordHandledException(final Throwable exception, final Map customSegmentation) { + Crashes target = target("[Crashes] recordUnhandledException"); + if (target != this) { + if (target != null) { + target.recordUnhandledException(exception, customSegmentation); + } + return; + } synchronized (Countly.instance()) { recordExceptionInternal(exception, false, customSegmentation, null); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleDeviceIdCore.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleDeviceIdCore.java index e77f93c5..ad413a17 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleDeviceIdCore.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleDeviceIdCore.java @@ -149,6 +149,10 @@ protected void changeDeviceIdInternal(String id, DeviceIdType type, boolean with return; } + if (!withMerge) { + stopLogGatheringForDeviceIdChange(); + } + internalConfig.storageProvider.setDeviceIdType(type.name()); internalConfig.storageProvider.setDeviceID(id); internalConfig.setDeviceId(new Config.DID(type.index, id)); @@ -156,6 +160,18 @@ protected void changeDeviceIdInternal(String id, DeviceIdType type, boolean with SDKCore.instance.notifyModulesDeviceIdChanged(old.id, withMerge); } + /** + * Has the gathered SDK log lines of the current user queued under the current device ID and + * the gather stopped, as the device ID is about to change without merge. + */ + private void stopLogGatheringForDeviceIdChange() { + SDKCore core = SDKCore.instance; + ModuleConfiguration configuration = core == null ? null : core.module(ModuleConfiguration.class); + if (configuration != null) { + configuration.stopLogGatheringForDeviceIdChange(); + } + } + /** * Gets id of the strategy supplied. In case strategy is not available, returns a fallback strategy. * In case strategy is available but id cannot be acquired right now, returns null. @@ -237,12 +253,32 @@ protected String getIDInternal() { public class DeviceId { + /** + * The interface a call goes to: this one while its module runs, the running module's when + * this one was handed out before that started or kept from one that stopped, and + * {@code null} while the feature does not run. + * + * @param call the call, for the log line + * @return the interface, {@code null} for none + */ + private DeviceId target(String call) { + ModuleDeviceIdCore running = runningInstance(call); + return running == null ? null : running.deviceIdInterface; + } + /** * Returns current device id. * * @return device id string */ public String getID() { + DeviceId target = target("[DeviceId] getID"); + if (target != this) { + if (target == null) { + return null; + } + return target.getID(); + } synchronized (Countly.instance()) { return getIDInternal(); } @@ -255,6 +291,13 @@ public String getID() { * @param newDeviceID device id to set */ public void setID(String newDeviceID) { + DeviceId target = target("[DeviceId] setID"); + if (target != this) { + if (target != null) { + target.setID(newDeviceID); + } + return; + } synchronized (Countly.instance()) { setIDInternal(newDeviceID); } @@ -266,6 +309,13 @@ public void setID(String newDeviceID) { * @return device id type */ public DeviceIdType getType() { + DeviceId target = target("[DeviceId] getType"); + if (target != this) { + if (target == null) { + return null; + } + return target.getType(); + } synchronized (Countly.instance()) { return getTypeInternal(); } @@ -277,6 +327,13 @@ public DeviceIdType getType() { * @param id new device id string, cannot be empty */ public void changeWithMerge(String id) { + DeviceId target = target("[DeviceId] changeWithMerge"); + if (target != this) { + if (target != null) { + target.changeWithMerge(id); + } + return; + } synchronized (Countly.instance()) { changeDeviceIdInternal(id, DeviceIdType.DEVELOPER_SUPPLIED, true); } @@ -288,6 +345,13 @@ public void changeWithMerge(String id) { * @param id new device id string, cannot be empty */ public void changeWithoutMerge(String id) { + DeviceId target = target("[DeviceId] changeWithoutMerge"); + if (target != this) { + if (target != null) { + target.changeWithoutMerge(id); + } + return; + } synchronized (Countly.instance()) { changeDeviceIdInternal(id, DeviceIdType.DEVELOPER_SUPPLIED, false); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleEvents.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleEvents.java index 1e48a875..9bcc528a 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleEvents.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleEvents.java @@ -1,18 +1,43 @@ package ly.count.sdk.java.internal; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import ly.count.sdk.java.Countly; import ly.count.sdk.java.Session; import ly.count.sdk.java.View; public class ModuleEvents extends ModuleBase { + /** + * The keys of the events the SDK records itself, which the custom event settings of the SDK + * behavior settings do not apply to. Any other key is a custom event, one starting with + * {@code [CLY]_} too. + */ + static final Set INTERNAL_EVENT_KEYS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + ModuleViews.KEY_VIEW_EVENT, FeedbackWidgetType.nps.eventKey, FeedbackWidgetType.survey.eventKey, FeedbackWidgetType.rating.eventKey))); + protected EventQueue eventQueue = null; final Map timedEvents = new ConcurrentHashMap<>(); protected Events eventsInterface = null; - ViewIdProvider viewIdProvider = null; IdGenerator idGenerator = null; String previousEventId = null; + /** + * Storage IDs of the event requests that carry a journey trigger and are waiting for their first + * response, which decides whether the content zone is refreshed. + */ + final Set journeyTriggerRequestIds = ConcurrentHashMap.newKeySet(); + /** + * The thread ending the timed events and the view of the previous device ID during a device ID + * change without merge; its sends wait, so those events go out in one request under that ID. + */ + private volatile Thread threadEndingEventsOfThePreviousDeviceId = null; @Override public void init(InternalConfig config) { @@ -25,12 +50,6 @@ public void init(InternalConfig config) { idGenerator = config.eventIdGenerator; } - @Override - public void initFinished(InternalConfig config) { - super.initFinished(config); - viewIdProvider = config.viewIdProvider; - } - @Override protected void onTimer() { addEventsToRequestQ(null); @@ -41,25 +60,77 @@ public Boolean onRequest(Request request) { return true; } + /** + * Refreshes the content zone once the server accepted a request that carries a journey trigger. + * The first attempt to send such a request settles it: when it fails, neither it nor its retry + * refreshes anything. A request dropped from the queue unsent settles it the same way. + * + * @param request the request that was sent + * @param response the response body, {@code null} when none could be read + * @param responseCode the response code, {@link Transport#NO_RESPONSE_CODE} when no response arrived + */ + @Override + public void onRequestCompleted(Request request, String response, int responseCode) { + if (request == null || !journeyTriggerRequestIds.remove(request.storageId())) { + return; + } + + if (!Transport.isSuccessfulResponse(responseCode, response)) { + L.d("[ModuleEvents] onRequestCompleted, the request [" + request.storageId() + "] carrying a journey trigger failed with code [" + responseCode + "], the content zone is not refreshed"); + return; + } + + SDKCore core = SDKCore.instance; + ModuleContent content = core == null ? null : core.module(ModuleContent.class); + if (content == null) { + L.d("[ModuleEvents] onRequestCompleted, the server accepted the request [" + request.storageId() + "] carrying a journey trigger, there is no content zone to refresh"); + return; + } + + L.d("[ModuleEvents] onRequestCompleted, the server accepted the request [" + request.storageId() + "] carrying a journey trigger, refreshing the content zone"); + content.refreshContentZoneInternal(); + } + + /** + * Settles the journey triggers carried by requests that were removed from the queue unread, + * which leaves their owner unknown and untold. No response will ever arrive for them, and none + * of them refreshes the content zone. + * + * @param requestIds storage IDs of the removed requests + */ + void onRequestsDroppedUnread(@Nonnull Collection requestIds) { + for (Long requestId : requestIds) { + if (journeyTriggerRequestIds.remove(requestId)) { + L.d("[ModuleEvents] onRequestsDroppedUnread, the request [" + requestId + "] carrying a journey trigger was dropped from the queue without being sent, the content zone is not refreshed"); + } + } + } + @Override public void deviceIdChanged(String oldDeviceId, boolean withMerge) { super.deviceIdChanged(oldDeviceId, withMerge); L.d("[ModuleEvents] deviceIdChanged: oldDeviceId = " + oldDeviceId + ", withMerge = " + withMerge); if (!withMerge) { - for (Map.Entry timedEventEntry : timedEvents.entrySet()) { - L.d("[ModuleEvents] deviceIdChanged, Ending timed event: [" + timedEventEntry.getKey() + "]"); - endEventInternal(timedEventEntry.getKey(), timedEventEntry.getValue().segmentation, timedEventEntry.getValue().count, timedEventEntry.getValue().sum); - } + //the new device ID is already set, so the events ended here are sent below under the old one, never on their own + threadEndingEventsOfThePreviousDeviceId = Thread.currentThread(); + try { + for (Map.Entry timedEventEntry : timedEvents.entrySet()) { + L.d("[ModuleEvents] deviceIdChanged, Ending timed event: [" + timedEventEntry.getKey() + "]"); + endEventInternal(timedEventEntry.getKey(), timedEventEntry.getValue().segmentation, timedEventEntry.getValue().count, timedEventEntry.getValue().sum); + } - // this part is to end and record the current view if exists - Session session = Countly.session(); - if (session != null && session.isActive()) { - View currentView = ((SessionImpl) session).currentView; - if (currentView != null) { - currentView.stop(true); - } else { - Storage.pushAsync(internalConfig, (SessionImpl) Countly.session()); + // this part is to end and record the current view if exists + Session session = Countly.session(); + if (session != null && session.isActive()) { + View currentView = ((SessionImpl) session).currentView; + if (currentView != null) { + currentView.stop(true); + } else { + Storage.pushAsync(internalConfig, (SessionImpl) Countly.session()); + } } + } finally { + threadEndingEventsOfThePreviousDeviceId = null; } addEventsToRequestQ(oldDeviceId); @@ -69,13 +140,30 @@ public void deviceIdChanged(String oldDeviceId, boolean withMerge) { @Override public void stop(InternalConfig config, final boolean clear) { super.stop(config, clear); + journeyTriggerRequestIds.clear(); if (clear) { eventQueue.clear(); timedEvents.clear(); } } - private synchronized void addEventsToRequestQ(String deviceId) { + /** + * Moves every queued event into one request. + * + * @param deviceId the device ID the events belong to, {@code null} for the current one + */ + private void addEventsToRequestQ(@Nullable String deviceId) { + addEventsToRequestQ(deviceId, false); + } + + /** + * Moves every queued event into one request. + * + * @param deviceId the device ID the events belong to, {@code null} for the current one + * @param journeyTrigger whether the events include a journey trigger, whose request refreshes + * the content zone once the server accepts it + */ + private synchronized void addEventsToRequestQ(@Nullable String deviceId, boolean journeyTrigger) { L.d("[ModuleEvents] addEventsToRequestQ"); if (eventQueue.getEQ().isEmpty()) { @@ -91,10 +179,28 @@ private synchronized void addEventsToRequestQ(String deviceId) { request.own(ModuleEvents.class); eventQueue.clear(); - ModuleRequests.pushAsync(internalConfig, request); + if (!journeyTrigger) { + ModuleRequests.pushAsync(internalConfig, request); + return; + } + + final Long requestId = request.storageId(); + journeyTriggerRequestIds.add(requestId); + L.d("[ModuleEvents] addEventsToRequestQ, the request [" + requestId + "] carries a journey trigger, the content zone is refreshed once the server accepts it"); + ModuleRequests.pushAsync(internalConfig, request, false, stored -> { + if (!Boolean.TRUE.equals(stored)) { + journeyTriggerRequestIds.remove(requestId); + } + }); } protected void recordEventInternal(String key, int count, Double sum, Double dur, Map segmentation, String eventIdOverride) { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + if (!configProvider.getTrackingEnabled()) { + L.d("[ModuleEvents] recordEventInternal, tracking is disabled by the SDK behavior settings, event [" + key + "] will not be recorded"); + return; + } + if (count <= 0) { L.w("[ModuleEvents] recordEventInternal, Count can't be less than 1, ignoring this event."); return; @@ -105,14 +211,34 @@ protected void recordEventInternal(String key, int count, Double sum, Double dur return; } + boolean isCustomEvent = !INTERNAL_EVENT_KEYS.contains(key); + if (isCustomEvent) { + if (!configProvider.getCustomEventTrackingEnabled()) { + L.d("[ModuleEvents] recordEventInternal, custom event tracking is disabled by the SDK behavior settings, event [" + key + "] will not be recorded"); + return; + } + + if (!UtilsListingFilters.applyEventFilter(key, configProvider)) { + L.w("[ModuleEvents] recordEventInternal, event [" + key + "] is filtered out by the event filter of the SDK behavior settings, it will not be recorded"); + return; + } + + segmentation = filteredCustomEventSegmentation(key, segmentation, configProvider); + } + + boolean journeyTrigger = isJourneyTrigger(key, isCustomEvent, segmentation, configProvider); + L.d("[ModuleEvents] recordEventInternal, Recording event with key: [" + key + "] and provided event ID of:[" + eventIdOverride + "] and segmentation with:[" + (segmentation == null ? "null" : segmentation.size()) + "] keys"); Utils.removeInvalidDataFromSegments(segmentation, L); - if (internalConfig.isAutoSendUserProperties() && internalConfig.sdk.userProfile() != null) { - internalConfig.sdk.module(ModuleUserProfile.class).saveInternal(); + if (isCustomEvent) { + key = UtilsInternalLimits.truncateKey(key, configProvider.getMaxKeyLength(), L, "[ModuleEvents] recordEventInternal"); + segmentation = UtilsInternalLimits.applySegmentationLimits(segmentation, configProvider, L, "[ModuleEvents] recordEventInternal"); } + ModuleUserProfile.saveIfSentAutomatically(internalConfig); + String eventId, pvid = null, cvid = null; if (Utils.isEmptyOrNull(eventIdOverride)) { L.d("[ModuleEvents] recordEventInternal, Generating new event id because it was null or empty"); @@ -121,6 +247,7 @@ protected void recordEventInternal(String key, int count, Double sum, Double dur eventId = eventIdOverride; } + ViewIdProvider viewIdProvider = internalConfig.viewIdProvider; if (key.equals(ModuleViews.KEY_VIEW_EVENT)) { pvid = viewIdProvider.getPreviousViewId(); } else { @@ -134,19 +261,86 @@ protected void recordEventInternal(String key, int count, Double sum, Double dur this.previousEventId = eventId; } - addEventToQueue(new EventImpl(key, count, sum, dur, segmentation, L, eventId, pvid, cvid, previousEventIdToSend)); + addEventToQueue(new EventImpl(key, count, sum, dur, segmentation, L, eventId, pvid, cvid, previousEventIdToSend), journeyTrigger); + } + + /** + * Applies the segmentation filter and then the segmentation filter of this event to the + * segmentation of a custom event. The filters work on a copy, as the map belongs to the caller. + * + * @param key the key of the custom event + * @param segmentation the segmentation the caller passed, {@code null} for none + * @param configProvider the settings in effect + * @return the filtered copy, {@code null} when there was no segmentation + */ + @Nullable + private Map filteredCustomEventSegmentation(@Nonnull String key, @Nullable Map segmentation, @Nonnull ConfigurationProvider configProvider) { + if (segmentation == null) { + return null; + } + + Map filtered = new HashMap<>(segmentation); + UtilsListingFilters.applySegmentationFilter(filtered, configProvider, L); + UtilsListingFilters.applyEventSegmentationFilter(key, filtered, configProvider, L); + return filtered; + } + + /** + * Whether an event is a journey trigger of the SDK behavior settings: a custom event whose key + * is in {@code jte}, or a view event whose name is in {@code jtv}. + * + * @param key the key of the event, as it was passed + * @param isCustomEvent whether the key is not one of {@link #INTERNAL_EVENT_KEYS} + * @param segmentation the segmentation of the event, {@code null} for none + * @param configProvider the settings in effect + * @return {@code true} for a journey trigger + */ + private static boolean isJourneyTrigger(@Nonnull String key, boolean isCustomEvent, @Nullable Map segmentation, @Nonnull ConfigurationProvider configProvider) { + if (isCustomEvent) { + return configProvider.getJourneyTriggerEvents().contains(key); + } + + if (!ModuleViews.KEY_VIEW_EVENT.equals(key) || segmentation == null) { + return false; + } + + Object viewName = segmentation.get(ModuleViews.KEY_NAME); + return viewName != null && configProvider.getJourneyTriggerViews().contains(viewName); } - private void addEventToQueue(EventImpl event) { + /** + * Adds an event to the event queue and sends the queue when it is full or holds a journey trigger. + * + * @param event the event + * @param journeyTrigger whether the event is a journey trigger + */ + private void addEventToQueue(@Nonnull EventImpl event, boolean journeyTrigger) { L.d("[ModuleEvents] addEventToQueue"); eventQueue.addEvent(event); - checkEventQueueToSend(false); + if (journeyTrigger) { + L.d("[ModuleEvents] addEventToQueue, event [" + event.key + "] is a journey trigger, sending the event queue now"); + } + checkEventQueueToSend(journeyTrigger, journeyTrigger); } void checkEventQueueToSend(boolean forceSend) { + checkEventQueueToSend(forceSend, false); + } + + /** + * Sends the queued events when asked to or when the queue reached its threshold. + * + * @param forceSend whether to send whatever is queued + * @param journeyTrigger whether the queue holds a journey trigger + */ + private void checkEventQueueToSend(boolean forceSend, boolean journeyTrigger) { + if (Thread.currentThread() == threadEndingEventsOfThePreviousDeviceId) { + L.d("[ModuleEvents] checkEventQueueToSend, the events of the previous device ID are being ended, they are sent together under that ID"); + return; + } L.d("[ModuleEvents] queue size:[" + eventQueue.eqSize() + "] || forceSend: " + forceSend); - if (forceSend || eventQueue.eqSize() >= internalConfig.getEventsBufferSize()) { - addEventsToRequestQ(null); + if (forceSend || eventQueue.eqSize() >= internalConfig.getConfigurationProvider().getEventQueueSizeThreshold()) { + addEventsToRequestQ(null, journeyTrigger); } } @@ -215,6 +409,19 @@ boolean cancelEventInternal(final String key) { public class Events { + /** + * The interface a call goes to: this one while its module runs, the running module's when + * this one was handed out before that started or kept from one that stopped, and + * {@code null} while the feature does not run. + * + * @param call the call, for the log line + * @return the interface, {@code null} for none + */ + private Events target(String call) { + ModuleEvents running = runningInstance(call); + return running == null ? null : running.eventsInterface; + } + /** * Record an event. * @@ -225,6 +432,13 @@ public class Events { * @param segmentation additional segmentation data that you want to set, leave null if you don't want to add anything */ public void recordEvent(String key, Map segmentation, int count, Double sum, Double dur) { + Events target = target("[Events] recordEvent"); + if (target != this) { + if (target != null) { + target.recordEvent(key, segmentation, count, sum, dur); + } + return; + } L.i("[Events] recordEvent: key = " + key + ", count = " + count + ", sum = " + sum + ", segmentation = " + segmentation + ", dur = " + dur); recordEventInternal(key, count, sum, dur, segmentation, null); } @@ -305,6 +519,10 @@ public void recordEvent(String key, int count, Double sum) { * @return true if no event with this key existed before and event is started, false otherwise */ public boolean startEvent(final String key) { + Events target = target("[Events] startEvent"); + if (target != this) { + return target != null && target.startEvent(key); + } L.i("[Events] startEvent: key = " + key); return startEventInternal(key); } @@ -316,6 +534,10 @@ public boolean startEvent(final String key) { * @return true if event with this key has been previously started, false otherwise */ public boolean endEvent(final String key) { + Events target = target("[Events] endEvent"); + if (target != this) { + return target != null && target.endEvent(key); + } L.i("[Events] endEvent: key = " + key); return endEventInternal(key, null, 1, null); } @@ -332,6 +554,10 @@ public boolean endEvent(final String key) { * @throws IllegalArgumentException if key is null or empty, count is less than 1, or if segmentation contains null or empty keys or values */ public boolean endEvent(final String key, final Map segmentation, final int count, final Double sum) { + Events target = target("[Events] endEvent"); + if (target != this) { + return target != null && target.endEvent(key, segmentation, count, sum); + } L.i("[Events] endEvent: key = " + key + ", segmentation = " + segmentation + ", count = " + count + ", sum = " + sum); return endEventInternal(key, segmentation, count, sum); } @@ -342,6 +568,10 @@ public boolean endEvent(final String key, final Map segmentation * @return true if event with this key has been previously started, false otherwise **/ public boolean cancelEvent(final String key) { + Events target = target("[Events] cancelEvent"); + if (target != this) { + return target != null && target.cancelEvent(key); + } L.i("[Events] cancelEvent: key = " + key); return cancelEventInternal(key); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleFeedback.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleFeedback.java index d088a526..f6702c0b 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleFeedback.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleFeedback.java @@ -263,7 +263,8 @@ private void reportFeedbackWidgetManuallyInternal(CountlyFeedbackWidget widgetIn } else { //widget was filled out //merge given segmentation - segm.putAll(widgetResult); + int maxValueSize = internalConfig.getConfigurationProvider().getMaxValueSize(); + segm.putAll(UtilsInternalLimits.truncateStringValues(widgetResult, maxValueSize, L, "[ModuleFeedback] reportFeedbackWidgetManuallyInternal")); } Countly.instance().events().recordEvent(widgetInfo.type.eventKey, segm); @@ -373,12 +374,34 @@ private String constructFeedbackWidgetUrlInternal(CountlyFeedbackWidget widgetIn public class Feedback { + /** + * The interface a call goes to: this one while its module runs, the running module's when + * this one was handed out before that started or kept from one that stopped, and + * {@code null} while the feature does not run. + * + * @param call the call, for the log line + * @return the interface, {@code null} for none + */ + private Feedback target(String call) { + ModuleFeedback running = runningInstance(call); + return running == null ? null : running.feedbackInterface; + } + /** * Get a list of available feedback widgets for this device ID * * @param callback retrieve widget list callback */ public void getAvailableFeedbackWidgets(@Nullable CallbackOnFinish> callback) { + Feedback target = target("[Feedback] getAvailableFeedbackWidgets"); + if (target != this) { + if (target != null) { + target.getAvailableFeedbackWidgets(callback); + } else if (callback != null) { + callback.onFinished(null, NOT_RUNNING_ERROR); + } + return; + } synchronized (Countly.instance()) { L.i("[Feedback] getAvailableFeedbackWidgets, Trying to retrieve feedback widget list"); @@ -392,6 +415,13 @@ public void getAvailableFeedbackWidgets(@Nullable CallbackOnFinish callback) { + Feedback target = target("[Feedback] getFeedbackWidgetData"); + if (target != this) { + if (target != null) { + target.getFeedbackWidgetData(widgetInfo, callback); + } else if (callback != null) { + callback.onFinished(null, NOT_RUNNING_ERROR); + } + return; + } synchronized (Countly.instance()) { L.i("[Feedback] getFeedbackWidgetData, Trying to retrieve feedback widget data"); @@ -447,6 +493,13 @@ public void getFeedbackWidgetData(@Nullable CountlyFeedbackWidget widgetInfo, @N * @param widgetResult widget result */ public void reportFeedbackWidgetManually(@Nullable CountlyFeedbackWidget widgetInfo, @Nullable JSONObject widgetData, @Nullable Map widgetResult) { + Feedback target = target("[Feedback] reportFeedbackWidgetManually"); + if (target != this) { + if (target != null) { + target.reportFeedbackWidgetManually(widgetInfo, widgetData, widgetResult); + } + return; + } synchronized (Countly.instance()) { L.i("[Feedback] reportFeedbackWidgetManually, Trying to report feedback widget manually"); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleLocation.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleLocation.java index c81b2adb..6f49fd4c 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleLocation.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleLocation.java @@ -3,6 +3,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import ly.count.sdk.java.Countly; +import org.json.JSONObject; public class ModuleLocation extends ModuleBase { boolean locationDisabled = false; @@ -28,23 +29,63 @@ public void init(InternalConfig internalConfig) { void disableLocationInternal() { L.d("[ModuleLocation] Calling 'disableLocationInternal'"); + clearLocation(); + sendLocation(); + } + + /** + * Forgets the location and marks it disabled, so the next location request erases it. + */ + private void clearLocation() { country = null; city = null; location = null; ip = null; locationDisabled = true; - sendLocation(); + } + + /** + * Queues a request carrying the location as it is now. + */ + private void pushLocation() { + ModuleRequests.pushAsync(internalConfig, new Request(prepareLocationParams()), true, null); + } + + /** + * Clears the location and erases it on the server right away, without waiting for a session to + * begin, as when the location consent is removed. + */ + void eraseLocation() { + L.d("[ModuleLocation] eraseLocation"); + clearLocation(); + pushLocation(); } void sendLocation() { L.d("[ModuleLocation] Calling 'sendLocation'"); SessionImpl session = internalConfig.sdk.getSession(); - if (session == null || session.getBegan() != null) { - ModuleRequests.pushAsync(internalConfig, new Request(prepareLocationParams()), true, null); + if (session == null || session.getBegan() != null || !isLocationSentWithSessionBegin()) { + pushLocation(); } // else case, values are added to the session begin request } + /** + * Whether a session that has not begun yet sends the location with its begin request, which the + * SDK behavior settings allow only while both session tracking and location tracking are enabled. + * + * @return {@code true} when the location can wait for the begin request + */ + private boolean isLocationSentWithSessionBegin() { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + return configProvider.getSessionTrackingEnabled() && configProvider.getLocationTrackingEnabled(); + } + void setLocationInternal(@Nullable String countryCode, @Nullable String cityName, @Nullable String gpsCoordinates, @Nullable String ipAddress) { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + if (!configProvider.getTrackingEnabled() || !configProvider.getLocationTrackingEnabled()) { + L.d("[ModuleLocation] setLocationInternal, location tracking disabled by SDK behavior settings; ignoring"); + return; + } L.d("[ModuleLocation] setLocationInternal, Setting location parameters, cc[" + countryCode + "] cy[" + city + "] gps[" + gpsCoordinates + "] ip[" + ipAddress + "]"); if (countryCode != null ^ city != null) { @@ -96,8 +137,10 @@ public void stop(InternalConfig internalConfig, boolean clear) { @Override public void initFinished(@Nonnull InternalConfig config) { if (config.isLocationDisabled()) { - //disable location if needed - disableLocationInternal(); + //disable location if needed, unless a settings response during init already did and sent the erase request + if (!locationDisabled) { + disableLocationInternal(); + } } else { //if we are not disabling location, check for other set values String[] locParams = config.getLocationParams(); // country, city, location, ip @@ -107,14 +150,35 @@ public void initFinished(@Nonnull InternalConfig config) { } } + /** + * Disables location when a server response turned location tracking off, which erases the + * location stored on the server as {@link Location#disableLocation()} does, and drops the + * location values held for the next user details request. + * + * @param config configuration of the running SDK + */ + @Override + protected void onSdkConfigurationChanged(InternalConfig config) { + synchronized (Countly.instance()) { + if (config.getConfigurationProvider().getLocationTrackingEnabled()) { + return; + } + dropDisallowedLegacyValues(); + if (!locationDisabled) { + L.d("[ModuleLocation] onSdkConfigurationChanged, location tracking was disabled by the SDK behavior settings, disabling location"); + disableLocationInternal(); + } + } + } + protected void saveLocationToParamsLegacy(Params params) { - if (countryLegacy != null) { + if (isLegacyValueAllowed(countryLegacy)) { params.add("country_code", countryLegacy); } - if (cityLegacy != null) { + if (isLegacyValueAllowed(cityLegacy)) { params.add("city", cityLegacy); } - if (locationLegacy != null) { + if (isLegacyValueAllowed(locationLegacy)) { params.add("location", locationLegacy); } countryLegacy = null; @@ -124,23 +188,79 @@ protected void saveLocationToParamsLegacy(Params params) { protected void setLocationLegacy(@Nullable Object countryCode, @Nullable Object city, @Nullable Object gpsCoordinates) { L.d("[Location] setLocationLegacy, calling legacy calls to send locations"); - if (countryCode != null) { + if (isLegacyValueAllowed(countryCode)) { countryLegacy = countryCode.toString(); } - if (city != null) { + if (isLegacyValueAllowed(city)) { cityLegacy = city.toString(); } - if (gpsCoordinates != null) { + if (isLegacyValueAllowed(gpsCoordinates)) { locationLegacy = gpsCoordinates.toString(); } } + /** + * Drops the held deprecated location values that may not be sent while location tracking is off, + * so a value set before the SDK behavior settings turned location tracking off is not sent after + * they turn it on again. + */ + private void dropDisallowedLegacyValues() { + if (!isLegacyValueAllowed(countryLegacy)) { + countryLegacy = null; + } + if (!isLegacyValueAllowed(cityLegacy)) { + cityLegacy = null; + } + if (!isLegacyValueAllowed(locationLegacy)) { + locationLegacy = null; + } + } + + /** + * Whether a deprecated location value can be held and sent with the user details. While the SDK + * behavior settings disable location tracking, only the JSON null that opts out of location + * services can, as it erases a location instead of tracking one. + * + * @param value the value, or {@code null} when none is set + * @return {@code true} when the value is set and can be sent + */ + private boolean isLegacyValueAllowed(@Nullable Object value) { + if (value == null) { + return false; + } + if (JSONObject.NULL.toString().equals(value.toString()) || internalConfig.getConfigurationProvider().getLocationTrackingEnabled()) { + return true; + } + L.d("[ModuleLocation] isLegacyValueAllowed, location tracking disabled by SDK behavior settings; dropping the location value"); + return false; + } + public class Location { + /** + * The interface a call goes to: this one while its module runs, the running module's when + * this one was handed out before that started or kept from one that stopped, and + * {@code null} while the feature does not run. + * + * @param call the call, for the log line + * @return the interface, {@code null} for none + */ + private Location target(String call) { + ModuleLocation running = runningInstance(call); + return running == null ? null : running.locationInterface; + } + /** * Disable sending of location data. Erases server side saved location information */ public void disableLocation() { + Location target = target("[Location] disableLocation"); + if (target != this) { + if (target != null) { + target.disableLocation(); + } + return; + } synchronized (Countly.instance()) { L.i("[Location] Calling 'disableLocation'"); @@ -159,6 +279,13 @@ public void disableLocation() { * @param ipAddress ipAddress like "192.168.88.33" */ public void setLocation(@Nullable String countryCode, @Nullable String city, @Nullable String gpsCoordinates, @Nullable String ipAddress) { + Location target = target("[Location] setLocation"); + if (target != this) { + if (target != null) { + target.setLocation(countryCode, city, gpsCoordinates, ipAddress); + } + return; + } synchronized (Countly.instance()) { L.i("[Location] Calling 'setLocation'"); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRemoteConfig.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRemoteConfig.java index 41abe3a7..8810618a 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRemoteConfig.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRemoteConfig.java @@ -1,8 +1,6 @@ package ly.count.sdk.java.internal; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; @@ -18,7 +16,6 @@ public class ModuleRemoteConfig extends ModuleBase { // if set to true we should add 'oi=1' to our RC download call boolean autoEnrollEnabled; boolean remoteConfigValuesShouldBeCached = false; - List downloadCallbacks = new ArrayList<>(2); ModuleRemoteConfig() { } @@ -36,7 +33,6 @@ public void init(final InternalConfig config) { remoteConfigValuesShouldBeCached = config.isRemoteConfigValueCachingEnabled(); autoEnrollEnabled = config.isAutoEnrollFlagEnabled(); - downloadCallbacks.addAll(config.getRemoteConfigGlobalCallbackList()); remoteConfigInterface = new RemoteConfig(); } @@ -251,7 +247,7 @@ private void cacheOrClearRCValuesIfNeeded() { } private void notifyDownloadCallbacks(final RCDownloadCallback devProvidedCallback, RequestResult requestResult, String message, boolean fullUpdate, Map downloadedValues) { - downloadCallbacks.forEach(callback -> callback.callback(requestResult, message, fullUpdate, downloadedValues)); + internalConfig.sdk.remoteConfigDownloadCallbacks.forEach(callback -> callback.callback(requestResult, message, fullUpdate, downloadedValues)); if (devProvidedCallback != null) { devProvidedCallback.callback(requestResult, message, fullUpdate, downloadedValues); @@ -309,6 +305,19 @@ public void stop(InternalConfig config, final boolean clear) { public class RemoteConfig { + /** + * The interface a call goes to: this one while its module runs, the running module's when + * this one was handed out before that started or kept from one that stopped, and + * {@code null} while the feature does not run. + * + * @param call the call, for the log line + * @return the interface, {@code null} for none + */ + private RemoteConfig target(String call) { + ModuleRemoteConfig running = runningInstance(call); + return running == null ? null : running.remoteConfigInterface; + } + /** * Manual remote config call that will initiate a download of all except the given remote config keys. * If no keys are provided then it will download all available RC values @@ -317,6 +326,15 @@ public class RemoteConfig { * @param callback This is called when the operation concludes */ public void downloadOmittingKeys(@Nullable String[] keysToOmit, @Nullable final RCDownloadCallback callback) { + RemoteConfig target = target("[RemoteConfig] downloadOmittingKeys"); + if (target != this) { + if (target != null) { + target.downloadOmittingKeys(keysToOmit, callback); + } else if (callback != null) { + callback.callback(RequestResult.Error, NOT_RUNNING_ERROR, keysToOmit == null || keysToOmit.length == 0, null); + } + return; + } synchronized (Countly.instance()) { L.i("[RemoteConfig] downloadOmittingKeys"); @@ -336,6 +354,15 @@ public void downloadOmittingKeys(@Nullable String[] keysToOmit, @Nullable final * @param callback This is called when the operation concludes */ public void downloadSpecificKeys(@Nullable String[] keysToInclude, @Nullable final RCDownloadCallback callback) { + RemoteConfig target = target("[RemoteConfig] downloadSpecificKeys"); + if (target != this) { + if (target != null) { + target.downloadSpecificKeys(keysToInclude, callback); + } else if (callback != null) { + callback.callback(RequestResult.Error, NOT_RUNNING_ERROR, keysToInclude == null || keysToInclude.length == 0, null); + } + return; + } synchronized (Countly.instance()) { L.i("[RemoteConfig] downloadSpecificKeys"); @@ -353,6 +380,15 @@ public void downloadSpecificKeys(@Nullable String[] keysToInclude, @Nullable fin * @param callback This is called when the operation concludes */ public void downloadAllKeys(@Nullable RCDownloadCallback callback) { + RemoteConfig target = target("[RemoteConfig] downloadAllKeys"); + if (target != this) { + if (target != null) { + target.downloadAllKeys(callback); + } else if (callback != null) { + callback.callback(RequestResult.Error, NOT_RUNNING_ERROR, true, null); + } + return; + } synchronized (Countly.instance()) { L.i("[RemoteConfig] downloadAllKeys"); updateRemoteConfigValues(null, null, callback); @@ -365,6 +401,13 @@ public void downloadAllKeys(@Nullable RCDownloadCallback callback) { * @return The available RC values */ public @Nonnull Map getValues() { + RemoteConfig target = target("[RemoteConfig] getValues"); + if (target != this) { + if (target == null) { + return new ConcurrentHashMap<>(); + } + return target.getValues(); + } synchronized (Countly.instance()) { L.i("[RemoteConfig] getValues"); return getRemoteConfigValueStoreInternal().getAllValues(); @@ -377,6 +420,13 @@ public void downloadAllKeys(@Nullable RCDownloadCallback callback) { * @return The available RC values */ public @Nonnull Map getAllValuesAndEnroll() { + RemoteConfig target = target("[RemoteConfig] getAllValuesAndEnroll"); + if (target != this) { + if (target == null) { + return new ConcurrentHashMap<>(); + } + return target.getAllValuesAndEnroll(); + } synchronized (Countly.instance()) { L.i("[RemoteConfig] getAllValuesAndEnroll"); Map values = getRemoteConfigValueStoreInternal().getAllValues(); @@ -399,6 +449,13 @@ public void downloadAllKeys(@Nullable RCDownloadCallback callback) { * @return The returned value. If no value existed for the key then the inner object (value) will be returned as "null" */ public @Nonnull RCData getValue(final @Nullable String key) { + RemoteConfig target = target("[RemoteConfig] getValue"); + if (target != this) { + if (target == null) { + return new RCData(null, true); + } + return target.getValue(key); + } synchronized (Countly.instance()) { L.i("[RemoteConfig] getValue, key:[" + key + "]"); @@ -418,6 +475,13 @@ public void downloadAllKeys(@Nullable RCDownloadCallback callback) { * @return The returned value. If no value existed for the key then the inner object will be returned as "null" */ public @Nonnull RCData getValueAndEnroll(@Nullable String key) { + RemoteConfig target = target("[RemoteConfig] getValueAndEnroll"); + if (target != this) { + if (target == null) { + return new RCData(null, true); + } + return target.getValueAndEnroll(key); + } synchronized (Countly.instance()) { L.i("[RemoteConfig] getValueAndEnroll, key:[" + key + "]"); @@ -443,6 +507,13 @@ public void downloadAllKeys(@Nullable RCDownloadCallback callback) { * @param keys - String array of keys (parameters) */ public void enrollIntoABTestsForKeys(@Nullable String[] keys) { + RemoteConfig target = target("[RemoteConfig] enrollIntoABTestsForKeys"); + if (target != this) { + if (target != null) { + target.enrollIntoABTestsForKeys(keys); + } + return; + } synchronized (Countly.instance()) { L.i("[RemoteConfig] enrollIntoABTestsForKeys"); if (keys == null || keys.length == 0) { @@ -459,6 +530,13 @@ public void enrollIntoABTestsForKeys(@Nullable String[] keys) { * @param keys - String array of keys (parameters) */ public void exitABTestsForKeys(@Nullable String[] keys) { + RemoteConfig target = target("[RemoteConfig] exitABTestsForKeys"); + if (target != this) { + if (target != null) { + target.exitABTestsForKeys(keys); + } + return; + } synchronized (Countly.instance()) { L.i("[RemoteConfig] exitABTestsForKeys"); if (keys == null) { @@ -469,18 +547,23 @@ public void exitABTestsForKeys(@Nullable String[] keys) { } /** - * Register a global callback for when download operations have finished + * Register a global callback for when download operations have finished. It is kept while + * remote config does not run, as before its consent is given, and called once it does. * * @param callback The callback that should be added */ public void registerDownloadCallback(@Nullable final RCDownloadCallback callback) { synchronized (Countly.instance()) { - L.i("[RemoteConfig] registerDownloadCallback"); + SDKCore core = SDKCore.instance; + if (core == null) { + return; + } + core.L.i("[RemoteConfig] registerDownloadCallback"); if (callback == null) { - L.w("[RemoteConfig] registerDownloadCallback, passed callback is null"); + core.L.w("[RemoteConfig] registerDownloadCallback, passed callback is null"); return; } - downloadCallbacks.add(callback); + core.remoteConfigDownloadCallbacks.add(callback); } } @@ -491,12 +574,16 @@ public void registerDownloadCallback(@Nullable final RCDownloadCallback callback */ public void removeDownloadCallback(@Nullable final RCDownloadCallback callback) { synchronized (Countly.instance()) { - L.i("[RemoteConfig] removeDownloadCallback"); + SDKCore core = SDKCore.instance; + if (core == null) { + return; + } + core.L.i("[RemoteConfig] removeDownloadCallback"); if (callback == null) { - L.w("[RemoteConfig] removeDownloadCallback, passed callback is null"); + core.L.w("[RemoteConfig] removeDownloadCallback, passed callback is null"); return; } - downloadCallbacks.remove(callback); + core.remoteConfigDownloadCallbacks.remove(callback); } } @@ -504,6 +591,13 @@ public void removeDownloadCallback(@Nullable final RCDownloadCallback callback) * Clear all stored remote config values. */ public void clearAll() { + RemoteConfig target = target("[RemoteConfig] clearAll"); + if (target != this) { + if (target != null) { + target.clearAll(); + } + return; + } synchronized (Countly.instance()) { L.i("[RemoteConfig] clearAll"); clearValueStoreInternal(); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRequests.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRequests.java index fcfb933b..9d047902 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRequests.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleRequests.java @@ -1,7 +1,15 @@ package ly.count.sdk.java.internal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** * Centralized place for all requests construction & handling. @@ -9,6 +17,9 @@ public class ModuleRequests extends ModuleBase { + //how long queueing a log batch waits for the storage, which never answers once it is shut down + static final long SDK_LOGS_STORE_TIMEOUT_MS = 10_000L; + private static Params metrics; public interface ParamsInjector { @@ -99,6 +110,11 @@ public static Future location(InternalConfig config, double latitude, d return null; } + if (!config.getConfigurationProvider().getLocationTrackingEnabled()) { + config.getLogger().d("[ModuleRequests] location, location tracking is disabled by the SDK behavior settings, the location is not sent"); + return null; + } + Request request = sessionRequest(config, null, null, null); request.params.add("location", latitude + "," + longitude); return pushAsync(config, request); @@ -191,21 +207,38 @@ public static Future pushAsync(InternalConfig config, Request request) } /** - * Common store-request logic: store & send a ping to the service. + * Common store-request logic: store & send a ping to the service. While the SDK behavior + * settings set the request queue size, storing a request drops the oldest ones the queue holds + * over that size. While the request carries a batch of gathered log lines, the storage thread is + * marked as doing their transport work until the callback returned, so nothing logged about + * queueing the batch is gathered. * * @param config InternalConfig to run in * @param request Request to store * @param noControl do not check empty validity of the request - * @param callback Callback (nullable) to call when storing is done, called in {@link Storage} {@link Thread} + * @param callback Callback (nullable) to call when storing is done, called in {@link Storage} {@link Thread}, + * or on the calling thread with {@code false} when the request is not stored * @return {@link Future} which resolves to {@code} true if stored successfully, false otherwise */ public static Future pushAsync(final InternalConfig config, final Request request, final boolean noControl, final Tasks.Callback callback) { config.getLogger().d("New request " + request.storageId() + ": " + request); + if (!config.getConfigurationProvider().getTrackingEnabled()) { + config.getLogger().d("[ModuleRequests] pushAsync, tracking disabled by SDK behavior settings; dropping request"); + if (callback != null) { + try { + callback.call(false); + } catch (Exception e) { + config.getLogger().e("[ModuleRequests] Exception in a callback " + e); + } + } + return null; + } + if (!noControl && request.isEmpty()) { if (callback != null) { try { - callback.call(null); + callback.call(false); } catch (Exception e) { config.getLogger().e("[ModuleRequests] Exception in a callback " + e); } @@ -216,11 +249,162 @@ public static Future pushAsync(final InternalConfig config, final Reque addRequiredTimeParametersToParams(request.params); addRequiredParametersToParams(config, request.params); + final boolean uploadsGatheredLogs = request.params.has(Log.keySdkLogs); return Storage.pushAsync(config, request, param -> { - SDKCore.instance.onRequest(config, request); - if (callback != null) { - callback.call(param); + Log logger = config.getLogger(); + boolean markedBefore = logger.isOwnTransportWork(); + logger.setOwnTransportWork(markedBefore || uploadsGatheredLogs); + try { + if (Boolean.TRUE.equals(param)) { + dropOldestRequestsOverQueueLimit(config); + } + SDKCore.instance.onRequest(config, request); + if (callback != null) { + callback.call(param); + } + } finally { + logger.setOwnTransportWork(markedBefore); } }); } + + /** + * Stores a request as it is, without the checks of {@link #pushAsync(InternalConfig, Request, boolean, Tasks.Callback)}, + * then drops the oldest requests the queue holds over the request queue size of the SDK behavior + * settings. Waits until both are done. + * + * @param config configuration of the running SDK + * @param request the request, with every parameter it is sent with + * @return whether the request was stored + */ + static boolean pushWithinQueueLimit(@Nonnull final InternalConfig config, @Nonnull final Request request) { + try { + return Boolean.TRUE.equals(Storage.pushAsync(config, request, stored -> { + if (Boolean.TRUE.equals(stored)) { + dropOldestRequestsOverQueueLimit(config); + } + }).get()); + } catch (InterruptedException | ExecutionException e) { + config.getLogger().e("[ModuleRequests] pushWithinQueueLimit, failed to store request " + request.storageId() + ", [" + e + "]"); + return false; + } + } + + /** + * Queues one batch of gathered SDK log lines as a request of its own, the batch as its + * {@code sdk_logs} parameter next to the required ones, through the checks of + * {@link #pushAsync(InternalConfig, Request)}, and waits until it is stored. When the wait times + * out or is interrupted, the store is cancelled, as a store left queued would still write the + * request after its lines were kept for another batch. + * + * @param config configuration of the running SDK + * @param payload the batch, as JSON text + * @return the storage ID of the stored request, {@code null} when it was refused, as while + * tracking is off, could not be stored or its store was cancelled + */ + @Nullable + static Long pushSdkLogs(@Nonnull InternalConfig config, @Nonnull String payload) { + Request request = nonSessionRequest(config); + request.params.add(Log.keySdkLogs, payload); + Future stored = null; + boolean queued; + try { + stored = pushAsync(config, request); + queued = stored != null && Boolean.TRUE.equals(stored.get(SDK_LOGS_STORE_TIMEOUT_MS, TimeUnit.MILLISECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + config.getLogger().w("[ModuleRequests] pushSdkLogs, interrupted while storing request " + request.storageId()); + queued = !cancelStore(config, request, stored); + } catch (TimeoutException e) { + config.getLogger().w("[ModuleRequests] pushSdkLogs, storing request " + request.storageId() + " took over [" + SDK_LOGS_STORE_TIMEOUT_MS + "] ms"); + queued = !cancelStore(config, request, stored); + } catch (ExecutionException | RuntimeException e) { + config.getLogger().w("[ModuleRequests] pushSdkLogs, failed to store request " + request.storageId() + ", [" + e + "]"); + queued = false; + } + return queued ? request.storageId() : null; + } + + /** + * Cancels the store of a log batch whose caller stopped waiting for it. A store that cannot be + * cancelled has already completed, so its batch counts as queued. + * + * @param config configuration of the running SDK + * @param request the request carrying the batch + * @param stored the store + * @return whether the store was cancelled, so its batch is not queued + */ + private static boolean cancelStore(@Nonnull InternalConfig config, @Nonnull Request request, @Nonnull Future stored) { + if (stored.cancel(false)) { + config.getLogger().w("[ModuleRequests] cancelStore, cancelled storing request " + request.storageId() + ", its batch is not queued"); + return true; + } + + config.getLogger().w("[ModuleRequests] cancelStore, storing request " + request.storageId() + " completed before it could be cancelled, its batch counts as queued"); + return false; + } + + /** + * Drops the oldest stored requests while the queue holds more than the request queue size of the + * SDK behavior settings. Without a size from the settings the queue has no limit. Must run on the + * storage thread, as it works on the stored files directly. The files are removed unread, so + * their owners are not told; the journey triggers they carried are settled instead. + * + * @param config configuration of the running SDK + * @return how many requests were dropped + */ + static int dropOldestRequestsOverQueueLimit(@Nonnull InternalConfig config) { + ConfigurationProvider configProvider = config.getConfigurationProvider(); + if (!configProvider.isRequestQueueMaxSizeFromBehaviorSettings()) { + return 0; + } + + List droppedIds = new ArrayList<>(); + try { + int maxSize = configProvider.getRequestQueueMaxSize(); + List requestIds = config.sdk.sdkStorage.storableList(config, Request.getStoragePrefix(), 0); + int overflow = requestIds.size() - maxSize; + if (overflow <= 0) { + return 0; + } + + Collections.sort(requestIds); + for (int i = 0; i < overflow; i++) { + Long requestId = requestIds.get(i); + if (Boolean.TRUE.equals(config.sdk.sdkStorage.storableRemove(config, new Request(requestId)))) { + droppedIds.add(requestId); + } + } + + config.getLogger().w("[ModuleRequests] dropOldestRequestsOverQueueLimit, the request queue went over its size of [" + maxSize + "] set by the SDK behavior settings, dropped the [" + droppedIds.size() + "] oldest requests"); + } catch (Exception e) { + config.getLogger().e("[ModuleRequests] dropOldestRequestsOverQueueLimit, failed to keep the request queue within its size, [" + e + "]"); + } + + settleJourneyTriggersOf(config, droppedIds); + return droppedIds.size(); + } + + /** + * Tells the events module which requests were dropped unread, so a journey trigger one of them + * carried stops waiting for a response. Only a set of IDs is touched, which is safe on the + * storage thread. + * + * @param config configuration of the running SDK + * @param droppedIds storage IDs of the dropped requests + */ + private static void settleJourneyTriggersOf(@Nonnull InternalConfig config, @Nonnull List droppedIds) { + if (droppedIds.isEmpty() || config.sdk == null) { + return; + } + + try { + ModuleEvents events = config.sdk.module(ModuleEvents.class); + if (events != null) { + events.onRequestsDroppedUnread(droppedIds); + } + } catch (Exception e) { + config.getLogger().e("[ModuleRequests] settleJourneyTriggersOf, failed to settle the journey triggers of the dropped requests, [" + e + "]"); + } + } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleSessions.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleSessions.java index 58977e5c..e4bfdcee 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleSessions.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleSessions.java @@ -39,6 +39,11 @@ public void initFinished(final InternalConfig config) { @Override protected void onTimer() { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + if (!configProvider.getTrackingEnabled() || !configProvider.getSessionTrackingEnabled()) { + L.d("[ModuleSessions] onTimer, session tracking disabled by SDK behavior settings; skipping update"); + return; + } if (!internalConfig.isBackendModeEnabled() && isActive() && getSession() != null) { L.i("[ModuleSessions] onTimer, updating session"); getSession().update(); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleUserProfile.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleUserProfile.java index 41aeb5da..796b5c63 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleUserProfile.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleUserProfile.java @@ -1,10 +1,18 @@ package ly.count.sdk.java.internal; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import ly.count.sdk.java.Countly; import ly.count.sdk.java.PredefinedUserPropertyKeys; import ly.count.sdk.java.User; @@ -15,6 +23,14 @@ public class ModuleUserProfile extends ModuleBase { static final String CUSTOM_KEY = "custom"; boolean isSynced = true; static final String PICTURE_BYTES = "[CLY]_picture_bytes"; + /** + * The keys of {@link PredefinedUserPropertyKeys}, which neither the user property filter nor the + * user property cache limit of the SDK behavior settings apply to. + */ + static final Set PREDEFINED_KEYS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + PredefinedUserPropertyKeys.NAME, PredefinedUserPropertyKeys.USERNAME, PredefinedUserPropertyKeys.EMAIL, + PredefinedUserPropertyKeys.ORGANIZATION, PredefinedUserPropertyKeys.PHONE, PredefinedUserPropertyKeys.PICTURE, + PredefinedUserPropertyKeys.PICTURE_PATH, PredefinedUserPropertyKeys.GENDER, PredefinedUserPropertyKeys.BIRTH_YEAR))); UserProfile userProfileInterface; private final Map sets; private final List ops; @@ -68,7 +84,7 @@ enum Op { } ModuleUserProfile() { - sets = new HashMap<>(); // keys should be nullable + sets = new LinkedHashMap<>(); // keys should be nullable, insertion order decides what the cache limit drops ops = new ArrayList<>(); } @@ -91,12 +107,23 @@ private Object optString(String key, Object value) { /** * Transforming changes in "sets" into a json contained in "changes" + *

+ * The SDK internal limits of the SDK behavior settings in effect apply here: string values of + * predefined properties, the picture excepted, are cut to the value size limit, and custom + * properties get the key length, value size and segmentation entry limits, the entry limit + * counting sets only, never modifications. * * @param changes JSONObject to store changes * @param params Params to store changes * @throws JSONException if something goes wrong */ void perform(JSONObject changes, Params params) throws JSONException { + ConfigurationProvider limits = internalConfig.getConfigurationProvider(); + int maxValueSize = limits.getMaxValueSize(); + Map customSets = supportedCustomSets(); + Map limitedCustomSets = UtilsInternalLimits.applySegmentationLimits(customSets, limits, L, "[ModuleUserProfile] perform"); + boolean customSetsLimited = !customSets.equals(limitedCustomSets); + for (String key : sets.keySet()) { Object value = sets.get(key); switch (key) { @@ -105,7 +132,7 @@ void perform(JSONObject changes, Params params) throws JSONException { case PredefinedUserPropertyKeys.EMAIL: case PredefinedUserPropertyKeys.ORGANIZATION: case PredefinedUserPropertyKeys.PHONE: - changes.put(key, optString(key, value)); + changes.put(key, optString(key, UtilsInternalLimits.truncateIfString(value, maxValueSize, L, "[ModuleUserProfile] perform"))); break; case PredefinedUserPropertyKeys.PICTURE: if (value == null) { @@ -137,6 +164,7 @@ void perform(JSONObject changes, Params params) throws JSONException { } break; case PredefinedUserPropertyKeys.GENDER: + value = UtilsInternalLimits.truncateIfString(value, maxValueSize, L, "[ModuleUserProfile] perform"); if (value == null || value instanceof User.Gender) { changes.put(PredefinedUserPropertyKeys.GENDER, value == null ? JSONObject.NULL : value.toString()); } else if (value instanceof String) { @@ -151,6 +179,7 @@ void perform(JSONObject changes, Params params) throws JSONException { } break; case PredefinedUserPropertyKeys.BIRTH_YEAR: + value = UtilsInternalLimits.truncateIfString(value, maxValueSize, L, "[ModuleUserProfile] perform"); if (value == null || value instanceof Integer) { changes.put(PredefinedUserPropertyKeys.BIRTH_YEAR, value == null ? JSONObject.NULL : value); } else if (value instanceof String) { @@ -164,25 +193,69 @@ void perform(JSONObject changes, Params params) throws JSONException { } break; default: - performCustomUpdate(key, value, changes); + if (!customSetsLimited || !customSets.containsKey(key)) { + performCustomUpdate(key, value, changes); + } break; } } - applyOps(changes); + if (customSetsLimited) { + for (Map.Entry entry : limitedCustomSets.entrySet()) { + performCustomUpdate(entry.getKey(), entry.getValue(), changes); + } + } + + applyOps(changes, limits); + } + + /** + * The pending sets of custom properties that {@link #performCustomUpdate} sends, the ones with a + * value of a supported type, in the order they were first set. + * + * @return the custom properties to send + */ + private Map supportedCustomSets() { + Map customSets = new LinkedHashMap<>(); + for (Map.Entry entry : sets.entrySet()) { + if (!isPredefinedKey(entry.getKey()) && isSupportedCustomValue(entry.getValue())) { + customSets.put(entry.getKey(), entry.getValue()); + } + } + return customSets; } - private void applyOps(final JSONObject changes) throws JSONException { + /** + * Whether a custom property can be sent with a value, which has to be of a supported type. + * + * @param value the value + * @return {@code true} for a supported type, {@code false} for any other type and for {@code null} + */ + private static boolean isSupportedCustomValue(@Nullable Object value) { + return value instanceof String || value instanceof Integer || value instanceof Float || value instanceof Double || value instanceof Boolean || value instanceof Object[]; + } + + /** + * Applies the pending modifications to the custom properties of the changes, with their keys cut + * to the key length limit and their string values cut to the value size limit. + * + * @param changes the changes to send + * @param limits the SDK behavior settings in effect + * @throws JSONException if a modification cannot be written + */ + private void applyOps(final JSONObject changes, @Nonnull ConfigurationProvider limits) throws JSONException { if (!ops.isEmpty() && !changes.has(CUSTOM_KEY)) { changes.put(CUSTOM_KEY, new JSONObject()); } for (OpParams opParam : ops) { - opParam.op.valueTransformer.apply(changes.getJSONObject(CUSTOM_KEY), opParam.key, opParam.value); + String key = UtilsInternalLimits.truncateKey(opParam.key, limits.getMaxKeyLength(), L, "[ModuleUserProfile] applyOps"); + Object value = UtilsInternalLimits.truncateIfString(opParam.value, limits.getMaxValueSize(), L, "[ModuleUserProfile] applyOps"); + opParam.op.valueTransformer.apply(changes.getJSONObject(CUSTOM_KEY), key, value); } } private void performCustomUpdate(final String key, final Object value, final JSONObject changes) throws JSONException { - if (value == null || value instanceof String || value instanceof Integer || value instanceof Float || value instanceof Double || value instanceof Boolean || value instanceof Object[]) { + if (value == null || isSupportedCustomValue(value)) { if (!changes.has(CUSTOM_KEY)) { changes.put(CUSTOM_KEY, new JSONObject()); } @@ -234,7 +307,14 @@ private void modifyCustomData(String key, Object value, Op mod) { L.w("[ModuleUserProfile] modifyCustomData, value is null, thus nothing to modify"); return; } + + if (!UtilsListingFilters.applyUserPropertyFilter(key, internalConfig.getConfigurationProvider())) { + L.w("[ModuleUserProfile] modifyCustomData, key [" + key + "] is filtered out by the user property filter of the SDK behavior settings, ignoring the modification"); + return; + } + ops.add(new OpParams(key, value, mod)); + applyCacheLimitToModifications(); isSynced = false; } @@ -250,14 +330,122 @@ protected void setPropertiesInternal(@Nonnull Map data) { return; } - sets.putAll(data); + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + Map accepted = new LinkedHashMap<>(); + for (Map.Entry entry : data.entrySet()) { + String key = entry.getKey(); + if (!isPredefinedKey(key) && !UtilsListingFilters.applyUserPropertyFilter(key, configProvider)) { + L.w("[ModuleUserProfile] setPropertiesInternal, key [" + key + "] is filtered out by the user property filter of the SDK behavior settings, it will not be set"); + continue; + } + accepted.put(key, entry.getValue()); + } + + if (accepted.isEmpty()) { + return; + } + + sets.putAll(accepted); + applyCacheLimitToCustomProperties(); isSynced = false; } + /** + * Whether a user property key is one of {@link PredefinedUserPropertyKeys}. + * + * @param key the key + * @return {@code true} for a predefined key + */ + private static boolean isPredefinedKey(@Nullable String key) { + return PREDEFINED_KEYS.contains(key); + } + + /** + * Keeps the pending custom properties within the user property cache limit of the SDK behavior + * settings by dropping the oldest ones. Predefined properties are not counted. + */ + private void applyCacheLimitToCustomProperties() { + int cacheLimit = internalConfig.getConfigurationProvider().getUserPropertyCacheLimit(); + if (sets.size() <= cacheLimit) { + return; + } + + int customCount = 0; + for (String key : sets.keySet()) { + if (!isPredefinedKey(key)) { + customCount++; + } + } + + int overflow = customCount - cacheLimit; + if (overflow <= 0) { + return; + } + + List dropped = new ArrayList<>(overflow); + Iterator keys = sets.keySet().iterator(); + while (keys.hasNext() && dropped.size() < overflow) { + String key = keys.next(); + if (!isPredefinedKey(key)) { + keys.remove(); + dropped.add(key); + } + } + + L.w("[ModuleUserProfile] applyCacheLimitToCustomProperties, [" + customCount + "] custom user properties are pending, over the cache limit of [" + cacheLimit + "] set by the SDK behavior settings, dropped the oldest ones: " + dropped); + } + + /** + * Keeps the keys with pending modifications within the user property cache limit of the SDK + * behavior settings by dropping every modification of the oldest keys. + */ + private void applyCacheLimitToModifications() { + int cacheLimit = internalConfig.getConfigurationProvider().getUserPropertyCacheLimit(); + if (ops.size() <= cacheLimit) { + return; + } + + Set modifiedKeys = new LinkedHashSet<>(); + for (OpParams op : ops) { + modifiedKeys.add(op.key); + } + + int overflow = modifiedKeys.size() - cacheLimit; + if (overflow <= 0) { + return; + } + + Set dropped = new LinkedHashSet<>(); + Iterator keys = modifiedKeys.iterator(); + while (dropped.size() < overflow) { + dropped.add(keys.next()); + } + ops.removeIf(op -> dropped.contains(op.key)); + + L.w("[ModuleUserProfile] applyCacheLimitToModifications, [" + modifiedKeys.size() + "] custom user properties have pending modifications, over the cache limit of [" + cacheLimit + "] set by the SDK behavior settings, dropped the modifications of the oldest ones: " + dropped); + } + + /** + * Sends the user properties set so far when the SDK sends them automatically, which it does + * before every session and event request. + * + * @param config the configuration of the running SDK + */ + static void saveIfSentAutomatically(@Nonnull InternalConfig config) { + if (!config.isAutoSendUserProperties()) { + return; + } + ModuleUserProfile module = config.sdk.module(ModuleUserProfile.class); + if (module != null) { + module.saveInternal(); + } + } + protected void saveInternal() { Params generatedParams = prepareRequestParamsForUserProfile(); - if (internalConfig.sdk.location() != null) { - internalConfig.sdk.module(ModuleLocation.class).saveLocationToParamsLegacy(generatedParams); + ModuleLocation location = internalConfig.sdk.runningModule(CoreFeature.Location, ModuleLocation.class); + if (location != null) { + location.saveLocationToParamsLegacy(generatedParams); } L.d("[ModuleUserProfile] saveInternal, generated params [" + generatedParams + "]"); @@ -266,8 +454,9 @@ protected void saveInternal() { return; } - if (internalConfig.isAutoSendUserProperties() && internalConfig.sdk.events() != null) { - internalConfig.sdk.module(ModuleEvents.class).checkEventQueueToSend(true); + ModuleEvents events = internalConfig.sdk.runningModule(CoreFeature.Events, ModuleEvents.class); + if (internalConfig.isAutoSendUserProperties() && events != null) { + events.checkEventQueueToSend(true); } ModuleRequests.pushAsync(internalConfig, new Request(generatedParams)); @@ -316,12 +505,32 @@ public void deviceIdChanged(String oldDeviceId, boolean withMerge) { public class UserProfile { + /** + * The interface a call goes to: this one while its module runs, the running module's when + * this one was handed out before that started or kept from one that stopped, and + * {@code null} while the feature does not run. + * + * @param call the call, for the log line + * @return the interface, {@code null} for none + */ + private UserProfile target(String call) { + ModuleUserProfile running = runningInstance(call); + return running == null ? null : running.userProfileInterface; + } + /** * Increment custom property value by 1. * * @param key String with property name to increment */ public void increment(String key) { + UserProfile target = target("[UserProfile] increment"); + if (target != this) { + if (target != null) { + target.increment(key); + } + return; + } synchronized (Countly.instance()) { modifyCustomData(key, 1, Op.INC); } @@ -334,6 +543,13 @@ public void increment(String key) { * @param value int value by which to increment */ public void incrementBy(String key, int value) { + UserProfile target = target("[UserProfile] incrementBy"); + if (target != this) { + if (target != null) { + target.incrementBy(key, value); + } + return; + } synchronized (Countly.instance()) { modifyCustomData(key, value, Op.INC); } @@ -346,6 +562,13 @@ public void incrementBy(String key, int value) { * @param value int value by which to multiply */ public void multiply(String key, double value) { + UserProfile target = target("[UserProfile] multiply"); + if (target != this) { + if (target != null) { + target.multiply(key, value); + } + return; + } synchronized (Countly.instance()) { modifyCustomData(key, value, Op.MUL); } @@ -358,6 +581,13 @@ public void multiply(String key, double value) { * @param value int value to check for max */ public void saveMax(String key, double value) { + UserProfile target = target("[UserProfile] saveMax"); + if (target != this) { + if (target != null) { + target.saveMax(key, value); + } + return; + } synchronized (Countly.instance()) { modifyCustomData(key, value, Op.MAX); } @@ -370,6 +600,13 @@ public void saveMax(String key, double value) { * @param value int value to check for min */ public void saveMin(String key, double value) { + UserProfile target = target("[UserProfile] saveMin"); + if (target != this) { + if (target != null) { + target.saveMin(key, value); + } + return; + } synchronized (Countly.instance()) { modifyCustomData(key, value, Op.MIN); } @@ -382,6 +619,13 @@ public void saveMin(String key, double value) { * @param value String value to set */ public void setOnce(String key, Object value) { + UserProfile target = target("[UserProfile] setOnce"); + if (target != this) { + if (target != null) { + target.setOnce(key, value); + } + return; + } synchronized (Countly.instance()) { modifyCustomData(key, value, Op.SET_ONCE); } @@ -395,6 +639,13 @@ public void setOnce(String key, Object value) { * @param value String with value to add to array */ public void push(String key, Object value) { + UserProfile target = target("[UserProfile] push"); + if (target != this) { + if (target != null) { + target.push(key, value); + } + return; + } synchronized (Countly.instance()) { modifyCustomData(key, value, Op.PUSH); } @@ -408,6 +659,13 @@ public void push(String key, Object value) { * @param value String with value to add to array */ public void pushUnique(String key, Object value) { + UserProfile target = target("[UserProfile] pushUnique"); + if (target != this) { + if (target != null) { + target.pushUnique(key, value); + } + return; + } synchronized (Countly.instance()) { modifyCustomData(key, value, Op.PUSH_UNIQUE); } @@ -421,6 +679,13 @@ public void pushUnique(String key, Object value) { * @param value String with value to remove from array */ public void pull(String key, Object value) { + UserProfile target = target("[UserProfile] pull"); + if (target != this) { + if (target != null) { + target.pull(key, value); + } + return; + } synchronized (Countly.instance()) { modifyCustomData(key, value, Op.PULL); } @@ -433,6 +698,13 @@ public void pull(String key, Object value) { * @param value the value for the user property to be set. The value should be the allowed data type. */ public void setProperty(String key, Object value) { + UserProfile target = target("[UserProfile] setProperty"); + if (target != this) { + if (target != null) { + target.setProperty(key, value); + } + return; + } synchronized (Countly.instance()) { L.i("[UserProfile] Calling 'setProperty'"); @@ -450,6 +722,13 @@ public void setProperty(String key, Object value) { * @param data Map of user properties to set */ public void setProperties(Map data) { + UserProfile target = target("[UserProfile] setProperties"); + if (target != this) { + if (target != null) { + target.setProperties(data); + } + return; + } synchronized (Countly.instance()) { L.i("[UserProfile] Calling 'setProperties'"); @@ -465,6 +744,13 @@ public void setProperties(Map data) { * Send provided values to server */ public void save() { + UserProfile target = target("[UserProfile] save"); + if (target != this) { + if (target != null) { + target.save(); + } + return; + } synchronized (Countly.instance()) { L.i("[UserProfile] Calling 'save'"); saveInternal(); @@ -475,6 +761,13 @@ public void save() { * Clear queued operations / modifications */ public void clear() { + UserProfile target = target("[UserProfile] clear"); + if (target != this) { + if (target != null) { + target.clear(); + } + return; + } synchronized (Countly.instance()) { L.i("[UserProfile] Calling 'clear'"); clearInternal(); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleViews.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleViews.java index dd176c3e..378c3f6c 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleViews.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleViews.java @@ -61,6 +61,9 @@ public void deviceIdChanged(String oldDeviceId, boolean withMerge) { @Override public void stop(InternalConfig config, boolean clear) { + if (config.viewIdProvider == this) { + config.viewIdProvider = SDKCore.NO_VIEW_IDS; + } viewsInterface = null; viewDataMap.clear(); if (globalViewSegmentation != null) { @@ -106,6 +109,7 @@ public void updateGlobalViewSegmentationInternal(@Nonnull Map se } private Map createViewEventSegmentation(@Nonnull ViewData vd, boolean firstView, boolean visit, Map customViewSegmentation) { + ConfigurationProvider limits = internalConfig.getConfigurationProvider(); Map viewSegmentation = new ConcurrentHashMap<>(); viewSegmentation.putAll(globalViewSegmentation); viewSegmentation.putAll(vd.viewSegmentation); @@ -114,7 +118,9 @@ private Map createViewEventSegmentation(@Nonnull ViewData vd, bo viewSegmentation.putAll(customViewSegmentation); } - viewSegmentation.put(KEY_NAME, vd.viewName); + viewSegmentation = applyInternalLimits(viewSegmentation, limits); + + viewSegmentation.put(KEY_NAME, UtilsInternalLimits.truncateKey(vd.viewName, limits.getMaxKeyLength(), L, "[ModuleViews] createViewEventSegmentation")); if (visit) { viewSegmentation.put(KEY_VISIT, KEY_VISIT_VALUE); } @@ -125,6 +131,30 @@ private Map createViewEventSegmentation(@Nonnull ViewData vd, bo return viewSegmentation; } + /** + * Applies the key length, value size and segmentation entry limits of the SDK behavior settings + * to the segmentation of a view event, before the SDK adds its own keys, which are neither cut + * nor counted. A key cut to one of the reserved keys is removed, and entries of unsupported + * types are removed before the entry limit counts, so they take no place under it. + * + * @param viewSegmentation the merged segmentation, which this module owns and may change + * @param limits the settings in effect + * @return the segmentation within the limits + */ + private Map applyInternalLimits(@Nonnull Map viewSegmentation, @Nonnull ConfigurationProvider limits) { + Map truncated = UtilsInternalLimits.truncateSegmentationKeysAndValues(viewSegmentation, limits.getMaxKeyLength(), limits.getMaxValueSize(), L, "[ModuleViews] createViewEventSegmentation"); + for (String reservedKey : reservedSegmentationKeysViews) { + if (truncated.remove(reservedKey) != null) { + L.w("[ModuleViews] createViewEventSegmentation, a key truncated to the reserved key [" + reservedKey + "] is removed"); + } + } + + if (truncated.size() > limits.getMaxSegmentationValues()) { + Utils.removeInvalidDataFromSegments(truncated, L); + } + return UtilsInternalLimits.limitSegmentationEntries(truncated, limits.getMaxSegmentationValues(), L, "[ModuleViews] createViewEventSegmentation"); + } + private void autoCloseRequiredViews(boolean closeAllViews, Map customViewSegmentation) { L.d("[ModuleViews] autoCloseRequiredViews"); List viewsToRemove = new ArrayList<>(); @@ -155,6 +185,11 @@ private void autoCloseRequiredViews(boolean closeAllViews, Map c * @return Returns link to Countly for call chaining */ @Nullable String startViewInternal(@Nullable String viewName, @Nullable Map customViewSegmentation, boolean viewShouldBeAutomaticallyStopped) { + ConfigurationProvider configProvider = internalConfig.getConfigurationProvider(); + if (!configProvider.getTrackingEnabled() || !configProvider.getViewTrackingEnabled()) { + L.d("[ModuleViews] startViewInternal, view tracking disabled by SDK behavior settings; ignoring"); + return null; + } if (viewName == null || viewName.isEmpty()) { L.e("[ModuleViews] startViewInternal, Trying to record view with null or empty view name, ignoring request"); @@ -208,6 +243,10 @@ void stopViewWithNameInternal(@Nullable String viewName, @Nullable Map customViewSegmentation) { + if (!isViewTrackingEnabled("stopViewWithIDInternal")) { + return; + } + ViewData vd = validateViewID(viewID, "stopViewWithIDInternal"); if (vd == null) { return; @@ -252,6 +291,10 @@ private void recordViewEndEvent(ViewData vd, @Nullable Map filte } void pauseViewWithIDInternal(String viewID) { + if (!isViewTrackingEnabled("pauseViewWithIDInternal")) { + return; + } + ViewData vd = validateViewID(viewID, "pauseViewWithIDInternal"); if (vd == null) { return; @@ -270,6 +313,10 @@ void pauseViewWithIDInternal(String viewID) { } void resumeViewWithIDInternal(String viewID) { + if (!isViewTrackingEnabled("resumeViewWithIDInternal")) { + return; + } + ViewData vd = validateViewID(viewID, "resumeViewWithIDInternal"); if (vd == null) { return; @@ -287,10 +334,29 @@ void resumeViewWithIDInternal(String viewID) { void stopAllViewsInternal(Map viewSegmentation) { L.d("[ModuleViews] stopAllViewsInternal"); + if (!isViewTrackingEnabled("stopAllViewsInternal")) { + return; + } autoCloseRequiredViews(true, viewSegmentation); } + /** + * Whether the SDK behavior settings allow view tracking. While they do not, stopping, pausing + * and resuming views is ignored, so every view keeps its state until view tracking is allowed again. + * + * @param function the ignored call, for the log + * @return {@code true} when view tracking is enabled + */ + private boolean isViewTrackingEnabled(@Nonnull String function) { + if (internalConfig.getConfigurationProvider().getViewTrackingEnabled()) { + return true; + } + + L.d("[ModuleViews] " + function + ", view tracking is disabled by the SDK behavior settings, ignoring the call"); + return false; + } + private ViewData validateViewID(String viewID, String function) { if (viewID == null || viewID.isEmpty()) { L.e("[ModuleViews] validateViewID, " + function + ", Trying to process view with null or empty view ID, ignoring request"); @@ -364,6 +430,19 @@ private void addSegmentationToViewWithIDInternal(String viewID, Map viewSegmentation) { + Views target = target("[Views] startAutoStoppedView"); + if (target != this) { + if (target == null) { + return null; + } + return target.startAutoStoppedView(viewName, viewSegmentation); + } synchronized (Countly.instance()) { L.i("[Views] Calling startAutoStoppedView [" + viewName + "] sg[" + viewSegmentation + "]"); return startViewInternal(viewName, viewSegmentation, true); @@ -402,6 +495,13 @@ public String startAutoStoppedView(@Nonnull String viewName, @Nullable Map viewSegmentation) { + Views target = target("[Views] startView"); + if (target != this) { + if (target == null) { + return null; + } + return target.startView(viewName, viewSegmentation); + } synchronized (Countly.instance()) { L.i("[Views] Calling startView vn[" + viewName + "] sg[" + viewSegmentation + "]"); return startViewInternal(viewName, viewSegmentation, false); @@ -428,6 +535,13 @@ public String startAutoStoppedView(@Nonnull String viewName, @Nullable Map - view segmentation */ public void stopViewWithName(@Nonnull String viewName, @Nullable Map viewSegmentation) { + Views target = target("[Views] stopViewWithName"); + if (target != this) { + if (target != null) { + target.stopViewWithName(viewName, viewSegmentation); + } + return; + } synchronized (Countly.instance()) { L.i("[Views] Calling stopViewWithName vn[" + viewName + "] sg[" + viewSegmentation + "]"); stopViewWithNameInternal(viewName, viewSegmentation); @@ -453,6 +574,13 @@ public void stopViewWithName(@Nonnull String viewName, @Nullable Map - view segmentation */ public void stopViewWithID(@Nonnull String viewID, @Nullable Map viewSegmentation) { + Views target = target("[Views] stopViewWithID"); + if (target != this) { + if (target != null) { + target.stopViewWithID(viewID, viewSegmentation); + } + return; + } synchronized (Countly.instance()) { L.i("[Views] Calling stopViewWithID vi[" + viewID + "] sg[" + viewSegmentation + "]"); stopViewWithIDInternal(viewID, viewSegmentation); @@ -478,6 +613,13 @@ public void stopViewWithID(@Nonnull String viewID, @Nullable Map * @param viewID String - view ID */ public void pauseViewWithID(@Nonnull String viewID) { + Views target = target("[Views] pauseViewWithID"); + if (target != this) { + if (target != null) { + target.pauseViewWithID(viewID); + } + return; + } synchronized (Countly.instance()) { L.i("[Views] Calling pauseViewWithID vi[" + viewID + "]"); pauseViewWithIDInternal(viewID); @@ -490,6 +632,13 @@ public void pauseViewWithID(@Nonnull String viewID) { * @param viewID String - view ID */ public void resumeViewWithID(@Nonnull String viewID) { + Views target = target("[Views] resumeViewWithID"); + if (target != this) { + if (target != null) { + target.resumeViewWithID(viewID); + } + return; + } synchronized (Countly.instance()) { L.i("[Views] Calling resumeViewWithID vi[" + viewID + "]"); resumeViewWithIDInternal(viewID); @@ -502,6 +651,13 @@ public void resumeViewWithID(@Nonnull String viewID) { * @param viewSegmentation Map - view segmentation */ public void stopAllViews(@Nullable Map viewSegmentation) { + Views target = target("[Views] stopAllViews"); + if (target != this) { + if (target != null) { + target.stopAllViews(viewSegmentation); + } + return; + } synchronized (Countly.instance()) { L.i("[Views] Calling stopAllViews sg[" + viewSegmentation + "]"); stopAllViewsInternal(viewSegmentation); @@ -515,6 +671,13 @@ public void stopAllViews(@Nullable Map viewSegmentation) { * @param viewSegmentation Map */ public void addSegmentationToViewWithName(@Nonnull String viewName, @Nullable Map viewSegmentation) { + Views target = target("[Views] addSegmentationToViewWithName"); + if (target != this) { + if (target != null) { + target.addSegmentationToViewWithName(viewName, viewSegmentation); + } + return; + } synchronized (Countly.instance()) { L.i("[Views] Calling addSegmentationToViewWithName vn[" + viewName + "] sg[" + viewSegmentation + "]"); addSegmentationToViewWithNameInternal(viewName, viewSegmentation); @@ -528,6 +691,13 @@ public void addSegmentationToViewWithName(@Nonnull String viewName, @Nullable Ma * @param viewSegmentation Map */ public void addSegmentationToViewWithID(@Nonnull String viewId, @Nullable Map viewSegmentation) { + Views target = target("[Views] addSegmentationToViewWithID"); + if (target != this) { + if (target != null) { + target.addSegmentationToViewWithID(viewId, viewSegmentation); + } + return; + } synchronized (Countly.instance()) { L.i("[Views] Calling addSegmentationToViewWithID vi[" + viewId + "] sg[" + viewSegmentation + "]"); addSegmentationToViewWithIDInternal(viewId, viewSegmentation); @@ -540,6 +710,13 @@ public void addSegmentationToViewWithID(@Nonnull String viewId, @Nullable Map - global view segmentation */ public void setGlobalViewSegmentation(@Nullable Map segmentation) { + Views target = target("[Views] setGlobalViewSegmentation"); + if (target != this) { + if (target != null) { + target.setGlobalViewSegmentation(segmentation); + } + return; + } synchronized (Countly.instance()) { L.i("[Views] Calling setGlobalViewSegmentation sg[" + (segmentation == null ? null : segmentation.size()) + "]"); @@ -553,6 +730,13 @@ public void setGlobalViewSegmentation(@Nullable Map segmentation * @param segmentation Map - global view segmentation */ public void updateGlobalViewSegmentation(@Nullable Map segmentation) { + Views target = target("[Views] updateGlobalViewSegmentation"); + if (target != this) { + if (target != null) { + target.updateGlobalViewSegmentation(segmentation); + } + return; + } synchronized (Countly.instance()) { L.i("[Views] Calling updateGlobalViewSegmentation sg[" + (segmentation == null ? null : segmentation.size()) + "]"); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKCore.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKCore.java index 3c8aa193..88ecd076 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKCore.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKCore.java @@ -6,8 +6,10 @@ import java.util.List; import java.util.Map; import java.util.Queue; -import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Function; import javax.annotation.Nonnull; import javax.annotation.Nullable; import ly.count.sdk.java.Config; @@ -21,23 +23,79 @@ public class SDKCore { protected Networking networking; protected Queue requestQueueMemory = null; protected final Object lockBRQStorage = new Object(); - private CountlyTimer countlyTimer; + private final Object lockTimer = new Object(); + //guarded by lockTimer for writes + volatile CountlyTimer countlyTimer; + private ModuleConfiguration moduleConfiguration; protected Log L = null; protected static ModuleBase testDummyModule = null;//set during testing when trying to check the SDK's lifecycle /** * Currently enabled features with consents */ - protected int consents = 0; + protected volatile int consents = 0; /** * Selected by config map of module mappings */ private static final Map> moduleMappings = new ConcurrentHashMap<>(); - // TreeMap to keep modules sorted by their feature indexes + /** + * The view IDs events carry while no views module runs. + */ + static final ViewIdProvider NO_VIEW_IDS = new ViewIdProvider() { + /** + * No view runs, so there is no current view ID. + */ + @Nonnull public String getCurrentViewId() { + return ""; + } + + /** + * No view runs, so there is no previous view ID. + */ + @Nonnull public String getPreviousViewId() { + return ""; + } + }; + + /** + * Interfaces of module instances the SDK never runs, handed out in place of the interface of a + * feature that does not run: before init, while its consent is not given, or when it is not + * enabled, so a caller never gets {@code null}. Like every module interface, each sends its + * calls to the module of its feature once that runs and ignores them until then. + */ + public static final ModuleEvents.Events dormantEvents = new ModuleEvents().new Events(); + public static final ModuleFeedback.Feedback dormantFeedback = new ModuleFeedback().new Feedback(); + public static final ModuleContent.Content dormantContent = new ModuleContent().new Content(); + public static final ModuleCrashes.Crashes dormantCrashes = new ModuleCrashes().new Crashes(); + public static final ModuleViews.Views dormantViews = new ModuleViews().new Views(); + public static final ModuleDeviceIdCore.DeviceId dormantDeviceId = new ModuleDeviceIdCore().new DeviceId(); + public static final ModuleRemoteConfig.RemoteConfig dormantRemoteConfig = new ModuleRemoteConfig().new RemoteConfig(); + public static final ModuleUserProfile.UserProfile dormantUserProfile = new ModuleUserProfile().new UserProfile(); + public static final ModuleLocation.Location dormantLocation = new ModuleLocation().new Location(); + + /** + * The remote config download callbacks, those of the configuration and those registered through + * the remote config interface, kept beyond the module, so one built when the consent is given, + * or given again, calls them too. + */ + final List remoteConfigDownloadCallbacks = new CopyOnWriteArrayList<>(); + + /** + * The display registered through the content interface, kept beyond the module, so one built + * when the consent is given, or given again, draws with it. + */ + volatile ContentDisplay contentDisplay = null; + + // sorted by feature index, and safe to iterate while a consent change adds or removes a module protected final Map modules; + /** + * Set once {@link #stop(boolean)} starts, after which no settings change reaches the modules. + */ + private volatile boolean stopping = false; + public enum Signal { DID(1), Crash(2), @@ -56,7 +114,7 @@ public int getIndex() { } public SDKCore() { - this.modules = new TreeMap<>(); + this.modules = new ConcurrentSkipListMap<>(); instance = this; sdkStorage = new SDKStorage(); } @@ -88,7 +146,15 @@ public boolean isTracking(Integer feat) { } private void onTimer() { - modules.forEach((feature, module) -> module.onTimer()); + modules.forEach((feature, module) -> { + try { + module.onTimer(); + } catch (Throwable t) { + //one failing module does not hold back the others on the same tick + L.e("[SDKCore] onTimer, " + module.getClass().getSimpleName() + " failed on a tick, [" + t + "]"); + } + }); + L.flushGatheredLogs(); } /** @@ -101,12 +167,20 @@ public void stop(final boolean clear) { if (instance == null) { return; } + stopping = true; if (networking != null) { networking.stop(config); } - countlyTimer.stopTimer(); + CountlyTimer timerToStop; + synchronized (lockTimer) { + timerToStop = countlyTimer; + countlyTimer = null; + } + if (timerToStop != null) { + timerToStop.stopTimer(); + } L.i("[SDKCore] Stopping Countly SDK" + (clear ? " and clearing all data" : "")); @@ -121,9 +195,11 @@ public void stop(final boolean clear) { modules.clear(); moduleMappings.clear(); + L.endLogGathering(!clear); sdkStorage.stop(config, clear);//from original super class user = null; + moduleConfiguration = null; config = null; instance = null; } @@ -166,6 +242,7 @@ public void onConsent(final int consent) { } consents = consents | (consent & config.getFeatures1()); + L.releaseConsentedLogLines(); for (CoreFeature feature : moduleMappings.keySet()) { ModuleBase existing = module(moduleMappings.get(feature)); @@ -176,6 +253,7 @@ public void onConsent(final int consent) { } else { module.init(config); module.initFinished(config); + module.setActive(true); modules.put(feature.getIndex(), module); } } @@ -201,14 +279,21 @@ public void onConsentRemoval(final InternalConfig config, int noConsent) { } if (removingConsent(noConsent, CoreFeature.Location)) { - user().edit().optOutFromLocationServices(); + ModuleLocation location = module(ModuleLocation.class); + if (location != null) { + location.eraseLocation(); + } } consents = consents & ~noConsent; + L.dropLinesWithoutConsent(); for (CoreFeature feature : moduleMappings.keySet()) { + if (feature == CoreFeature.Sessions || (noConsent & feature.getIndex()) == 0) { + continue; + } ModuleBase existing = module(moduleMappings.get(feature)); - if (feature.getIndex() != CoreFeature.Sessions.getIndex() && existing != null) { + if (existing != null) { existing.stop(config, true); modules.remove(feature.getIndex()); } @@ -256,6 +341,10 @@ protected void buildModules(InternalConfig config, int features) throws IllegalA // standard required internal features modules.put(-3, new ModuleDeviceIdCore()); modules.put(-2, new ModuleRequests()); + if (moduleConfiguration != null) { + modules.put(-1, moduleConfiguration); + } + modules.put(0, new ModuleConnectionTest()); modules.put(CoreFeature.Sessions.getIndex(), new ModuleSessions()); modules.put(CoreFeature.UserProfiles.getIndex(), new ModuleUserProfile()); @@ -405,10 +494,6 @@ public void init(final InternalConfig givenConfig) { //setup module mapping prepareMappings(); - //create internal timer - countlyTimer = new CountlyTimer(L); - countlyTimer.startTimer(config.getSendUpdateEachSeconds(), this::onTimer); - //setup and perform migrations MigrationHelper migrationHelper = new MigrationHelper(L); migrationHelper.setupMigrations(config.storageProvider); @@ -418,7 +503,20 @@ public void init(final InternalConfig givenConfig) { setDeviceIdFromStorageIfExist(config); + //resolve the SDK behavior settings before anything reads them + moduleConfiguration = new ModuleConfiguration(); + moduleConfiguration.init(config); + applyLoggingSetting(config, moduleConfiguration); + applyConsentRequirementAtInit(config, moduleConfiguration.getConsentRequired()); + L.setLogGatheringProviders(moduleConfiguration, this::givenLogLineConsents, gatheredLogBatchQueue); + + //create internal timer + synchronized (lockTimer) { + startTimer(moduleConfiguration.getSessionUpdateInterval()); + } + requestQueueMemory = new ArrayDeque<>(config.getRequestQueueMaxSize()); + remoteConfigDownloadCallbacks.addAll(config.getRemoteConfigGlobalCallbackList()); if (config.viewIdGenerator == null) { config.viewIdGenerator = Utils::safeRandomVal; @@ -429,15 +527,7 @@ public void init(final InternalConfig givenConfig) { } if (config.viewIdProvider == null) { - config.viewIdProvider = new ViewIdProvider() { - @Nonnull public String getCurrentViewId() { - return ""; - } - - @Nonnull public String getPreviousViewId() { - return ""; - } - }; + config.viewIdProvider = NO_VIEW_IDS; } // ModuleSessions is always enabled, even without consent @@ -449,7 +539,9 @@ public void init(final InternalConfig givenConfig) { modules.forEach((feature, module) -> { try { - module.init(config); + if (module != moduleConfiguration) { + module.init(config); + } module.setActive(true); } catch (IllegalArgumentException | IllegalStateException e) { L.e("[SDKCore] Error during module initialization" + e); @@ -528,6 +620,144 @@ private void checkNetworking(InternalConfig config) { private void initFinished(final InternalConfig config) { modules.forEach((feature, module) -> module.initFinished(config)); checkNetworking(config); + L.onSdkInitFinished(); + } + + /** + * The features whose consent lets a gathered log line go: every feature while no consent is + * required, otherwise the ones consent was given for. Read inside log calls, so it never logs. + * + * @return the {@link CoreFeature} indexes, every bit while no consent is required + */ + private int givenLogLineConsents() { + InternalConfig current = config; + if (current == null) { + return 0; + } + if (!current.requiresConsent()) { + return ~0; + } + int given = consents; + //a feature the app never enabled holds no data and can never get consent, so it holds no line back + return given == 0 ? 0 : given | (~current.getFeatures1() & Log.UNATTRIBUTED_LINE_CONSENT); + } + + /** + * The request queue as the log gathering uses it: one request per batch of gathered log lines, + * waiting until it is sent or dropped. + */ + private final Log.LogBatchQueue gatheredLogBatchQueue = new Log.LogBatchQueue() { + /** + * Stores the batch as a request of its own, unless the SDK stopped. + */ + @Nullable + @Override + public Long queueLogBatch(@Nonnull String payload) { + InternalConfig current = config; + return current == null ? null : ModuleRequests.pushSdkLogs(current, payload); + } + + /** + * Whether the request file is still stored, which it is until the request is sent or dropped. + */ + @Override + public boolean isQueued(@Nonnull Long requestId) { + InternalConfig current = config; + return current != null && sdkStorage.storableExists(current, Request.getStoragePrefix(), requestId); + } + }; + + /** + * Makes the consent requirement the SDK behavior settings resolved the one every module reads + * through {@link InternalConfig#requiresConsent()}. It is applied here, before the modules are + * built, and only here: a value that arrives later takes effect on the next init. + * + * @param config configuration of the SDK being initialized + * @param consentRequired the resolved consent requirement + */ + private void applyConsentRequirementAtInit(@Nonnull InternalConfig config, boolean consentRequired) { + if (consentRequired != config.requiresConsent()) { + L.i("[SDKCore] applyConsentRequirementAtInit, SDK behavior settings set the consent requirement to [" + consentRequired + "]"); + config.setRequiresConsent(consentRequired); + } + } + + /** + * Starts the global timer, replacing the running one. The caller holds {@link #lockTimer}. + * + * @param intervalSeconds seconds between two ticks + */ + private void startTimer(int intervalSeconds) { + countlyTimer = new CountlyTimer(L); + countlyTimer.startTimer(intervalSeconds, this::onTimer); + } + + /** + * Restarts the global timer when its interval differs from the given one. Never waits for the + * replaced timer, as a settings change can be applied from a timer task. + * + * @param intervalSeconds the session update interval now in effect + */ + private void restartTimerIfIntervalChanged(int intervalSeconds) { + synchronized (lockTimer) { + CountlyTimer runningTimer = countlyTimer; + if (runningTimer == null || runningTimer.getTimerDelaySeconds() == intervalSeconds) { + return; + } + + L.d("[SDKCore] restartTimerIfIntervalChanged, session update interval changed from [" + runningTimer.getTimerDelaySeconds() + "] to [" + intervalSeconds + "] seconds, restarting the timer"); + runningTimer.stopTimer(false); + startTimer(intervalSeconds); + } + } + + /** + * Applies the {@code log} switch of the SDK behavior settings to what the SDK logger prints. + * + * @param config configuration whose logger prints + * @param provider the settings in effect + */ + private void applyLoggingSetting(@Nonnull InternalConfig config, @Nonnull ConfigurationProvider provider) { + Log logger = config.getLogger(); + if (logger == null) { + return; + } + + logger.setLoggingEnabled(provider.getLoggingEnabled()); + logger.d("[SDKCore] applyLoggingSetting, logging enabled:[" + provider.getLoggingEnabled() + "], printing from level:[" + logger.getPrintLevel() + "]"); + } + + /** + * Applies SDK behavior settings that a server response changed while the SDK runs: sets what the + * logger prints, restarts the global timer when the session update interval changed, lets the + * request queue drain when networking is allowed, then notifies every module, each on its own. + * A change that arrives once the SDK started stopping is ignored, as its modules are being stopped. + * + * @param changedConfig configuration whose settings changed + */ + void onSdkConfigurationChanged(@Nonnull InternalConfig changedConfig) { + if (stopping || config == null || config != changedConfig) { + L.d("[SDKCore] onSdkConfigurationChanged, the SDK is stopping or was stopped before the change arrived, ignoring it"); + return; + } + + L.i("[SDKCore] onSdkConfigurationChanged"); + ConfigurationProvider provider = changedConfig.getConfigurationProvider(); + applyLoggingSetting(changedConfig, provider); + restartTimerIfIntervalChanged(provider.getSessionUpdateInterval()); + + if (provider.getNetworkingEnabled()) { + checkNetworking(changedConfig); + } + + List modulesToNotify = new ArrayList<>(modules.values()); + for (ModuleBase module : modulesToNotify) { + try { + module.onSdkConfigurationChanged(changedConfig); + } catch (Exception e) { + L.e("[SDKCore] onSdkConfigurationChanged, " + module.getClass().getSimpleName() + " failed to apply the change, [" + e + "]"); + } + } } public UserImpl user() { @@ -642,8 +872,8 @@ public Boolean isRequestReady(Request request) { if (cls == null) { return true; } else { + // The owner stays on the request: Transport.send reads it to hand over the response, then removes it before sending ModuleBase module = module(cls); - request.params.remove(Request.MODULE); if (module == null) { return true; } else { @@ -655,17 +885,23 @@ public Boolean isRequestReady(Request request) { /** * After a network request has been finished * propagate that response to the module - * that owns the request + * that owns the request. Neither looking the owner up nor the owner failing to handle the + * response ever fails the send. * * @param request the request that was sent, used to identify the request */ public void onRequestCompleted(Request request, String response, int responseCode, Class requestOwner) { - if (requestOwner != null) { - ModuleBase module = module(requestOwner); + if (requestOwner == null) { + return; + } + try { + ModuleBase module = module(requestOwner); if (module != null) { module.onRequestCompleted(request, response, responseCode); } + } catch (Exception e) { + L.e("[SDKCore] onRequestCompleted, " + requestOwner.getSimpleName() + " failed to handle the response of request [" + (request == null ? null : request.storageId()) + "], [" + e + "]"); } } @@ -723,7 +959,7 @@ private boolean processCrash(InternalConfig config, Long id) { ModuleRequests.addRequiredParametersToParams(config, request.params); ModuleRequests.addRequiredTimeParametersToParams(request.params); - if (Storage.push(config, request)) { + if (ModuleRequests.pushWithinQueueLimit(config, request)) { L.i("[SDKCore] Added request " + request.storageId() + " instead of crash " + crash.storageId()); checkNetworking(config); Boolean success = Storage.remove(config, crash); @@ -749,113 +985,120 @@ TimedEvents timedEvents() { return ((ModuleSessions) module(CoreFeature.Sessions)).timedEvents(); } + /** + * The events interface: the running module's, or one that ignores calls until the events feature runs, as its consent is not given or it is not enabled. + * + * @return the interface, never {@code null} + */ public ModuleEvents.Events events() { - if (!hasConsentForFeature(CoreFeature.Events)) { - L.v("[SDKCore] events, Events feature has no consent, returning null"); - return null; - } - - ModuleEvents module = module(ModuleEvents.class); - if (module == null) { - return null; - } - - return module.eventsInterface; + return runningInterface(CoreFeature.Events, ModuleEvents.class, module -> module.eventsInterface, dormantEvents); } + /** + * The feedback interface: the running module's, or one that ignores calls until the feedback feature runs, as its consent is not given or it is not enabled. + * + * @return the interface, never {@code null} + */ public ModuleFeedback.Feedback feedback() { - - if (!hasConsentForFeature(CoreFeature.Feedback)) { - L.v("[SDKCore] feedback, Feedback feature has no consent, returning null"); - return null; - } - - ModuleFeedback module = module(ModuleFeedback.class); - if (module == null) { - // Consent is not the only gate: the feature may simply never have been enabled, - // in which case the module was never built and there is nothing to hand back. - L.v("[SDKCore] feedback, feedback feature was not enabled, returning null"); - return null; - } - - return module.feedbackInterface; + return runningInterface(CoreFeature.Feedback, ModuleFeedback.class, module -> module.feedbackInterface, dormantFeedback); } + /** + * The content interface: the running module's, or one that ignores calls until the content feature runs, as its consent is not given or it is not enabled. + * + * @return the interface, never {@code null} + */ public ModuleContent.Content content() { - if (!hasConsentForFeature(CoreFeature.Content)) { - L.v("[SDKCore] content, Content feature has no consent, returning null"); - return null; - } - - ModuleContent module = module(ModuleContent.class); - if (module == null) { - return null; - } - - return module.contentInterface; + return runningInterface(CoreFeature.Content, ModuleContent.class, module -> module.contentInterface, dormantContent); } + /** + * The crashes interface: the running module's, or one that ignores calls until crash reporting runs, as its consent is not given or it is not enabled. + * + * @return the interface, never {@code null} + */ public ModuleCrashes.Crashes crashes() { - if (!hasConsentForFeature(CoreFeature.CrashReporting)) { - L.v("[SDKCore] crash, Crash Reporting feature has no consent, returning null"); - return null; - } - - ModuleCrashes module = module(ModuleCrashes.class); - if (module == null) { - // Consent is not the only gate: the feature may simply never have been enabled, - // in which case the module was never built and there is nothing to hand back. - L.v("[SDKCore] crashes, crashes feature was not enabled, returning null"); - return null; - } - - return module.crashInterface; + return runningInterface(CoreFeature.CrashReporting, ModuleCrashes.class, module -> module.crashInterface, dormantCrashes); } + /** + * The views interface: the running module's, or one that ignores calls until the views feature runs, as its consent is not given or it is not enabled. + * + * @return the interface, never {@code null} + */ public ModuleViews.Views views() { - ModuleViews module = module(ModuleViews.class); - if (module == null) { - L.v("[SDKCore] views, Views feature has no consent, returning null"); - return null; - } - - return module.viewsInterface; + return runningInterface(null, ModuleViews.class, module -> module.viewsInterface, dormantViews); } + /** + * The device ID interface: the running module's, or one that ignores calls until the device ID module runs. + * + * @return the interface, never {@code null} + */ public ModuleDeviceIdCore.DeviceId deviceId() { - return module(ModuleDeviceIdCore.class).deviceIdInterface; + return runningInterface(null, ModuleDeviceIdCore.class, module -> module.deviceIdInterface, dormantDeviceId); } + /** + * The remote config interface: the running module's, or one that ignores calls until remote config runs, as its consent is not given or it is not enabled. + * + * @return the interface, never {@code null} + */ public ModuleRemoteConfig.RemoteConfig remoteConfig() { - if (!hasConsentForFeature(CoreFeature.RemoteConfig)) { - L.v("[SDKCore] remoteConfig, RemoteConfig feature has no consent, returning null"); - return null; - } - - ModuleRemoteConfig module = module(ModuleRemoteConfig.class); - if (module == null) { - // Consent is not the only gate: the feature may simply never have been enabled, - // in which case the module was never built and there is nothing to hand back. - L.v("[SDKCore] remoteConfig, remoteConfig feature was not enabled, returning null"); - return null; - } - - return module.remoteConfigInterface; + return runningInterface(CoreFeature.RemoteConfig, ModuleRemoteConfig.class, module -> module.remoteConfigInterface, dormantRemoteConfig); } + /** + * The user profile interface: the running module's, or one that ignores calls until the user profile module runs. + * + * @return the interface, never {@code null} + */ public ModuleUserProfile.UserProfile userProfile() { - return module(ModuleUserProfile.class).userProfileInterface; + return runningInterface(null, ModuleUserProfile.class, module -> module.userProfileInterface, dormantUserProfile); } + /** + * The location interface: the running module's, or one that ignores calls until the location feature runs, as its consent is not given or it is not enabled. + * + * @return the interface, never {@code null} + */ public ModuleLocation.Location location() { - if (!hasConsentForFeature(CoreFeature.Location)) { - L.v("[SDKCore] location, Location feature has no consent, returning null"); - return null; - } - ModuleLocation module = module(ModuleLocation.class); - if (module == null) { - return null; + return runningInterface(CoreFeature.Location, ModuleLocation.class, module -> module.locationInterface, dormantLocation); + } + + /** + * The interface of a running module, or the one handed out while it does not run, whose calls + * go to the module once it runs and are ignored until then. + * + * @param consent the feature whose consent the module needs, {@code null} when it needs none + * @param moduleClass the class of the module + * @param interfaceOf reads the interface of a module + * @param dormant the interface handed out while the module does not run + * @param the module type + * @param the interface type + * @return the interface, never {@code null} + */ + private I runningInterface(@Nullable CoreFeature consent, @Nonnull Class moduleClass, @Nonnull Function interfaceOf, @Nonnull I dormant) { + M module = consent == null ? module(moduleClass) : runningModule(consent, moduleClass); + I running = module == null ? null : interfaceOf.apply(module); + if (running == null) { + L.v("[SDKCore] runningInterface, " + moduleClass.getSimpleName() + " does not run, handing out an interface that ignores calls until it does"); + return dormant; } - return module.locationInterface; + return running; + } + + /** + * The module of a feature while it runs: it was built and the feature's consent is given. + * + * @param feature the feature + * @param moduleClass the class of its module + * @param the module type + * @return the module, {@code null} while it does not run + */ + @Nullable + M runningModule(@Nonnull CoreFeature feature, @Nonnull Class moduleClass) { + M module = module(moduleClass); + return module != null && hasConsentForFeature(feature) ? module : null; } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKStorage.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKStorage.java index 4575608e..0f5814a8 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKStorage.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/SDKStorage.java @@ -33,6 +33,7 @@ public class SDKStorage implements StorageProvider { protected static final String key_device_id_type = "did_t"; protected static final String key_remote_config = "rc"; protected static final String key_migration_version = "dv"; + protected static final String key_server_config = "sc"; private JsonFileStorage jsonFileStorage; @@ -170,6 +171,11 @@ private FileOutputStream openFileAsOutputStream(ly.count.sdk.java.internal.Inter } public byte[] storableReadBytes(InternalConfig config, String filename) { + //every storable starts without a file, so a missing one is no error + if (!new File(createFileFullPath(config, filename)).exists()) { + return null; + } + ByteArrayOutputStream buffer = null; FileInputStream stream = null; @@ -188,6 +194,7 @@ public byte[] storableReadBytes(InternalConfig config, String filename) { return buffer.toByteArray(); } catch (FileNotFoundException e) { + L.e("[SDKStorage] Error while opening file " + filename + " " + e); return null; } catch (IOException e) { L.e("[SDKStorage] Error while reading file " + filename + " " + e); @@ -276,6 +283,18 @@ private String[] getFileList(InternalConfig config, Predicate fileValidati return fileNames.toArray(ret); } + /** + * Whether the file of a storable exists, as a stored request does until it is sent or dropped. + * + * @param config configuration of the running SDK + * @param prefix the storage prefix of the storable + * @param id the storage ID of the storable + * @return {@code true} while the file exists + */ + boolean storableExists(InternalConfig config, String prefix, Long id) { + return new File(createFileFullPath(config, getName(prefix, id.toString()))).exists(); + } + public List storableList(InternalConfig config, String prefix, int slice) { if (Utils.isEmptyOrNull(prefix)) { L.e("[SDKStorage] Cannot get list of ids without prefix"); @@ -386,6 +405,20 @@ public void setMigrationVersion(Integer migrationVersion) { jsonFileStorage.addAndSave(key_migration_version, migrationVersion); } + @Override + public void setServerConfig(String config) { + if (config == null) { + jsonFileStorage.deleteAndSave(key_server_config); + } else { + jsonFileStorage.addAndSave(key_server_config, config); + } + } + + @Override + public String getServerConfig() { + return jsonFileStorage.getString(key_server_config); + } + @Override public boolean isCountlyStorageEmpty() { String[] files = getCountlyFileList(config); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/SessionImpl.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/SessionImpl.java index 78e53d57..b4e881f7 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/SessionImpl.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/SessionImpl.java @@ -107,6 +107,8 @@ Future begin(Long now) { if (SDKCore.instance == null) { L.e("[SessionImpl] Countly is not initialized"); return null; + } else if (!isSessionTrackingEnabled("begin")) { + return null; } else if (began != null) { L.e("[SessionImpl] Session already began"); return null; @@ -118,15 +120,14 @@ Future begin(Long now) { } this.consents = SDKCore.instance.consents; - if (config.isAutoSendUserProperties() && config.sdk.userProfile() != null) { - config.sdk.module(ModuleUserProfile.class).saveInternal(); - } + ModuleUserProfile.saveIfSentAutomatically(config); if (pushOnChange) { Storage.pushAsync(config, this); } - if (hasConsent(CoreFeature.Location) && config.sdk.module(ModuleLocation.class) != null) { - params.add(config.sdk.module(ModuleLocation.class).prepareLocationParams()); + ModuleLocation locationModule = config.sdk.module(ModuleLocation.class); + if (hasConsent(CoreFeature.Location) && locationModule != null && config.getConfigurationProvider().getLocationTrackingEnabled()) { + params.add(locationModule.prepareLocationParams()); } Future ret = ModuleRequests.sessionBegin(config, this); @@ -151,6 +152,8 @@ Future update(Long now) { if (SDKCore.instance == null) { L.e("[SessionImpl] Countly is not initialized"); return null; + } else if (!isSessionTrackingEnabled("update")) { + return null; } else if (began == null) { L.e("[SessionImpl] Session is not began to update it"); return null; @@ -160,9 +163,7 @@ Future update(Long now) { } this.consents = SDKCore.instance.consents; - if (config.isAutoSendUserProperties() && config.sdk.userProfile() != null) { - config.sdk.module(ModuleUserProfile.class).saveInternal(); - } + ModuleUserProfile.saveIfSentAutomatically(config); Long duration = updateDuration(now); @@ -188,6 +189,8 @@ Future end(Long now, final Tasks.Callback callback, String did if (SDKCore.instance == null) { L.e("[SessionImpl] Countly is not initialized"); return null; + } else if (!isSessionTrackingEnabled("end")) { + return null; } else if (began == null) { L.e("[SessionImpl] Session is not began to end it"); return null; @@ -198,9 +201,7 @@ Future end(Long now, final Tasks.Callback callback, String did ended = now == null ? System.nanoTime() : now; this.consents = SDKCore.instance.consents; - if (config.isAutoSendUserProperties() && config.sdk.userProfile() != null) { - config.sdk.module(ModuleUserProfile.class).saveInternal(); - } + ModuleUserProfile.saveIfSentAutomatically(config); if (currentView != null) { currentView.stop(true); @@ -222,10 +223,22 @@ Future end(Long now, final Tasks.Callback callback, String did return ret; } + /** + * Ends a session left on disk by an earlier run, or removes it when there is nothing to end. + * While session tracking is disabled by the SDK behavior settings, no session request may be + * sent, so an unended session is removed without an end request: keeping it would end it on a + * later run with a stale duration. + * + * @param config configuration of the SDK being initialized + * @return whether the session was ended or removed, {@code null} when it is left as it is + */ Boolean recover(InternalConfig config) { Log L = config.getLogger(); if ((System.currentTimeMillis() - id) < 0) { return null; + } else if (!config.getConfigurationProvider().getSessionTrackingEnabled()) { + L.d("[SessionImpl] recover, session tracking is disabled by the SDK behavior settings, removing session " + id + " without ending it"); + return Storage.remove(config, this); } else { Future future; if (began == null) { @@ -260,6 +273,22 @@ public boolean isActive() { return began != null && ended == null; } + /** + * Whether the SDK behavior settings allow session tracking. While they do not, beginning, + * updating and ending a session are ignored, and the session keeps its state. + * + * @param function the ignored call, for the log + * @return {@code true} when session tracking is enabled + */ + private boolean isSessionTrackingEnabled(String function) { + if (config.getConfigurationProvider().getSessionTrackingEnabled()) { + return true; + } + + L.d("[SessionImpl] " + function + ", session tracking is disabled by the SDK behavior settings, ignoring the call"); + return false; + } + /** * Calculate time since last {@link #update()} or since {@link #begin()} if no {@link #update()} yet made, * set {@link #updated} to now. @@ -308,11 +337,11 @@ protected TimedEvents timedEvents() { */ @Override public void recordEvent(Event event) { - L.d("[SessionImpl] recordEvent: " + event.toString()); if (!SDKCore.enabled(CoreFeature.Events)) { L.i("[SessionImpl] recordEvent: Skipping event - feature is not enabled"); return; } + L.d("[SessionImpl] recordEvent"); if (began == null) { begin(); @@ -368,11 +397,11 @@ public Session addLocation(double latitude, double longitude) { return this; } - L.d("[SessionImpl] addLocation: latitude = " + latitude + " longitude = " + longitude); if (!SDKCore.enabled(CoreFeature.Location)) { L.i("[SessionImpl] addLocation: Skipping event - feature is not enabled"); return this; } + L.d("[SessionImpl] addLocation"); Countly.instance().location().setLocation(null, null, latitude + "," + longitude, null); return this; } @@ -386,11 +415,11 @@ public Session addLocation(double latitude, double longitude) { * @deprecated use {@link ModuleViews.Views#startView(String)} instead via {@link Countly#views()} */ public View view(String name, boolean start) { - L.d("[SessionImpl] view: name = " + name + " start = " + start); if (!SDKCore.enabled(CoreFeature.Views)) { L.i("[SessionImpl] view: Skipping view - feature is not enabled"); return null; } + L.d("[SessionImpl] view: start = " + start); if (currentView != null) { currentView.stop(false); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/Storage.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/Storage.java index 255529b3..1add3b9b 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/Storage.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/Storage.java @@ -228,8 +228,11 @@ public static Future readAsync(final InternalConfig conf public T call() throws Exception { Boolean done = config.sdk.sdkStorage.storableRead(config, storable); T ret = null; - if (done == null || !done) { - config.getLogger().e("[Storage] No data for file " + name(storable)); + if (done == null) { + //every storable starts without a file, and one that could not be read was logged as an error already + config.getLogger().d("[Storage] No data for file " + name(storable)); + } else if (!done) { + config.getLogger().e("[Storage] Could not restore the data of file " + name(storable)); } else { ret = storable; } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/StorageProvider.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/StorageProvider.java index 1983dd7a..da2d7bd4 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/StorageProvider.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/StorageProvider.java @@ -69,4 +69,18 @@ public interface StorageProvider { * @return true if empty, false otherwise */ boolean isCountlyStorageEmpty(); + + /** + * Persist the most-recently-validated SDK behavior settings JSON. + * + * @param config serialized configuration object, or {@code null} to clear + */ + void setServerConfig(String config); + + /** + * Read the persisted SDK behavior settings JSON. + * + * @return serialized configuration, or {@code null} if none persisted yet + */ + String getServerConfig(); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java index 19650e24..315d7e9f 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java @@ -30,6 +30,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.Future; +import java.util.function.LongConsumer; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; @@ -52,6 +55,12 @@ //class Network extends ModuleBase { - may be public class Transport implements X509TrustManager { + /** + * The response code handed to the module that owns a request when an attempt to send it ended + * before any response arrived. + */ + static final int NO_RESPONSE_CODE = -1; + private Log L = null; private static final String PARAMETER_TAMPERING_DIGEST = "SHA-256"; private static final String CHECKSUM = "checksum256"; @@ -144,21 +153,12 @@ HttpURLConnection connection(final Request request) throws IOException { connection.setConnectTimeout(1000 * config.getNetworkConnectionTimeout()); connection.setReadTimeout(1000 * config.getNetworkReadTimeout()); - if (connection instanceof HttpsURLConnection && sslContext != null) { - HttpsURLConnection https = (HttpsURLConnection) connection; - https.setSSLSocketFactory(sslContext.getSocketFactory()); - } + usePinningSocketFactory(connection); if (config.getCustomNetworkRequestHeaders() != null) { //if there are custom header values, add them L.v("[Transport] connection, Adding [" + config.getCustomNetworkRequestHeaders().size() + "] custom header fields"); - for (Map.Entry entry : config.getCustomNetworkRequestHeaders().entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - if (key != null && value != null && !key.isEmpty()) { - connection.addRequestProperty(key, value); - } - } + addCustomRequestHeaders(connection); } if (!usingGET) { @@ -221,6 +221,80 @@ HttpURLConnection connection(final Request request) throws IOException { return connection; } + /** + * Opens the GET of one connection test probe: a bare request, built as + * {@link #openBareGetConnection(String, int)} builds it, that verifies the server against the + * configured pins and carries the custom request headers, as every queued request does. Nothing + * is sent before the response is read. + * + * @param url the probe URL + * @param timeoutMs the connect and the read timeout, in milliseconds + * @return the connection, not connected yet + * @throws IOException when the connection cannot be opened + */ + HttpURLConnection openProbeConnection(@Nonnull String url, int timeoutMs) throws IOException { + HttpURLConnection connection = openBareGetConnection(url, timeoutMs); + usePinningSocketFactory(connection); + addCustomRequestHeaders(connection); + return connection; + } + + /** + * Opens a GET that carries nothing but its URL, follows no redirect and uses no cache. The + * redirect setting is made on this connection only, never through the default of every + * connection. + * + * @param url the URL + * @param timeoutMs the connect and the read timeout, in milliseconds + * @return the connection, not connected yet + * @throws IOException when the connection cannot be opened + */ + static HttpURLConnection openBareGetConnection(@Nonnull String url, int timeoutMs) throws IOException { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setRequestMethod("GET"); + connection.setInstanceFollowRedirects(false); + connection.setUseCaches(false); + connection.setConnectTimeout(timeoutMs); + connection.setReadTimeout(timeoutMs); + connection.setDoInput(true); + connection.setDoOutput(false); + return connection; + } + + /** + * Makes an HTTPS connection verify the server against the configured pins, when pins are + * configured. + * + * @param connection the connection, not connected yet + */ + private void usePinningSocketFactory(HttpURLConnection connection) { + if (connection instanceof HttpsURLConnection && sslContext != null) { + HttpsURLConnection https = (HttpsURLConnection) connection; + https.setSSLSocketFactory(sslContext.getSocketFactory()); + } + } + + /** + * Adds the custom request headers of the configuration, skipping any with a {@code null} or + * empty name or a {@code null} value. + * + * @param connection the connection, not connected yet + */ + private void addCustomRequestHeaders(HttpURLConnection connection) { + Map headers = config.getCustomNetworkRequestHeaders(); + if (headers == null) { + return; + } + + for (Map.Entry entry : headers.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key != null && value != null && !key.isEmpty()) { + connection.addRequestProperty(key, value); + } + } + } + void addMultipart(OutputStream output, PrintWriter writer, final String boundary, final String contentType, final String name, final String value, final byte[] file) throws IOException { writer.append("--").append(boundary).append(Utils.CRLF); if (file != null) { @@ -300,27 +374,58 @@ String response(HttpURLConnection connection) { } } + /** + * Sends a request once and hands its response to the module that owns it. While the request + * carries a batch of gathered log lines, the sending thread is marked as doing their transport + * work, so nothing logged about the upload is gathered. + * + * @param request request to send + * @return the task sending the request, resolving to whether the server accepted it + */ public Tasks.Task send(final Request request) { + return send(request, null); + } + + /** + * Sends a request as {@link #send(Request)} does, and tells how long the server took to answer + * it: from opening the connection, which sends the request, to reading the whole response. + * + * @param request request to send + * @param responseTimeListener given the response time in milliseconds once a response was read, + * never called when none was; {@code null} for none + * @return the task sending the request, resolving to whether the server accepted it + */ + Tasks.Task send(final Request request, @Nullable final LongConsumer responseTimeListener) { return new Tasks.Task(request.storageId()) { @Override public Boolean call() { - return send(); + L.setOwnTransportWork(request.params.has(Log.keySdkLogs)); + try { + return send(); + } finally { + L.setOwnTransportWork(false); + } } public Boolean send() { L.i("[network] [send] Sending request: " + request); HttpURLConnection connection = null; + Class requestOwner = null; try { - Class requestOwner = request.owner(); + requestOwner = request.owner(); request.params.remove(Request.MODULE); + long sendStartNs = System.nanoTime(); connection = connection(request); connection.connect(); int code = connection.getResponseCode(); String response = response(connection); + if (responseTimeListener != null) { + responseTimeListener.accept((System.nanoTime() - sendStartNs) / 1_000_000L); + } try { if (request.params.has(Params.PARAM_OLD_DEVICE_ID) || request.params.has("token_session")) { @@ -341,6 +446,7 @@ public Boolean send() { return processResponse(code, response, request.storageId()); } catch (IOException e) { L.w("[network] Error while sending request " + request + " " + e); + notifyOwnerOfAttemptWithoutResponse(request, requestOwner); return false; } catch (Exception e) { L.e("[network] Unexpected error while sending request " + request + " " + e); @@ -354,6 +460,57 @@ public Boolean send() { }; } + /** + * Tells the module that owns a request that an attempt to send it ended before a response + * arrived, with a {@code null} response and {@link #NO_RESPONSE_CODE}. + * + * @param request the request + * @param requestOwner the module that owns it, {@code null} for none + */ + private void notifyOwnerOfAttemptWithoutResponse(Request request, Class requestOwner) { + SDKCore core = SDKCore.instance; + if (core == null || requestOwner == null) { + return; + } + + try { + core.onRequestCompleted(request, null, NO_RESPONSE_CODE, requestOwner); + } catch (Exception e) { + L.e("[network] Failed to tell the owner of request [" + request.storageId() + "] that it got no response, [" + e + "]"); + } + } + + /** + * Whether a response means the server accepted the request: a 2xx code and a JSON object body + * that has a {@code result}, the rule {@link #processResponse(int, String, Long)} applies. + * + * @param code the response code + * @param response the response body, {@code null} when none arrived + * @return {@code true} when the request was accepted + */ + static boolean isSuccessfulResponse(int code, String response) { + if (response == null) { + return false; + } + + try { + return isSuccessfulResponse(code, new JSONObject(response)); + } catch (Exception e) { + return false; + } + } + + /** + * Whether a parsed response means the server accepted the request. + * + * @param code the response code + * @param response the parsed response body + * @return {@code true} for a 2xx code with a {@code result} + */ + private static boolean isSuccessfulResponse(int code, JSONObject response) { + return code >= 200 && code < 300 && response.has("result"); + } + Boolean processResponse(int code, String response, Long requestId) { L.i("[network] [processResponse] Code [" + code + "] response [" + response + "] for request[" + requestId + "]"); @@ -364,7 +521,7 @@ Boolean processResponse(int code, String response, Long requestId) { try { JSONObject jsonObject = new JSONObject(response); - if (code >= 200 && code < 300 && jsonObject.has("result")) { + if (isSuccessfulResponse(code, jsonObject)) { L.d("[network] Success"); return true; } else { diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/UserEditorImpl.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/UserEditorImpl.java index 33e586cc..ca8eda24 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/UserEditorImpl.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/UserEditorImpl.java @@ -1,6 +1,7 @@ package ly.count.sdk.java.internal; import java.io.File; +import java.util.function.Consumer; import ly.count.sdk.java.Countly; import ly.count.sdk.java.PredefinedUserPropertyKeys; import ly.count.sdk.java.User; @@ -100,68 +101,70 @@ public UserEditor setBirthyear(String birthyear) { @Override public UserEditor setLocale(String locale) { - L.d("setLocale: locale = " + locale); + L.d("[UserEditorImpl] setLocale"); return this; } @Override public UserEditor setCountry(String country) { - L.d("setCountry: country = " + country); + L.d("[UserEditorImpl] setCountry"); String countryLegacy = country == null ? JSONObject.NULL.toString() : country; - callLocationService(() -> SDKCore.instance.module(ModuleLocation.class).setLocationLegacy(countryLegacy, null, null)); + callLocationService(module -> module.setLocationLegacy(countryLegacy, null, null)); return this; } @Override public UserEditor setCity(String city) { - L.d("setCity: city = " + city); + L.d("[UserEditorImpl] setCity"); String cityLegacy = city == null ? JSONObject.NULL.toString() : city; - callLocationService(() -> SDKCore.instance.module(ModuleLocation.class).setLocationLegacy(null, cityLegacy, null)); + callLocationService(module -> module.setLocationLegacy(null, cityLegacy, null)); return this; } @Override public UserEditor setLocation(String location) { - L.d("setLocation: location = " + location); + L.d("[UserEditorImpl] setLocation"); if (location != null) { String[] comps = location.split(","); if (comps.length == 2) { try { String locationString = Double.parseDouble(comps[0]) + "," + Double.parseDouble(comps[1]); - callLocationService(() -> SDKCore.instance.module(ModuleLocation.class).setLocationLegacy(null, null, locationString)); + callLocationService(module -> module.setLocationLegacy(null, null, locationString)); return this; } catch (Throwable t) { - L.e("[UserEditorImpl] Invalid location format: " + location + " " + t); + L.e("[UserEditorImpl] setLocation, invalid location format, " + t.getClass().getSimpleName()); return this; } } else { - L.e("[UserEditorImpl] Invalid location format: " + location); + L.e("[UserEditorImpl] setLocation, invalid location format"); return this; } } else { - callLocationService(() -> SDKCore.instance.module(ModuleLocation.class).setLocationLegacy(null, null, JSONObject.NULL)); + callLocationService(module -> module.setLocationLegacy(null, null, JSONObject.NULL)); return this; } } @Override public UserEditor setLocation(double latitude, double longitude) { - L.d("setLocation: latitude = " + latitude + " longitude" + longitude); + L.d("[UserEditorImpl] setLocation"); String locationString = latitude + "," + longitude; - callLocationService(() -> SDKCore.instance.module(ModuleLocation.class).setLocationLegacy(null, null, locationString)); + callLocationService(module -> module.setLocationLegacy(null, null, locationString)); return this; } @Override public UserEditor optOutFromLocationServices() { L.d("optOutFromLocationServices"); - callLocationService(() -> SDKCore.instance.module(ModuleLocation.class).setLocationLegacy(JSONObject.NULL, JSONObject.NULL, JSONObject.NULL)); + callLocationService(module -> module.setLocationLegacy(JSONObject.NULL, JSONObject.NULL, JSONObject.NULL)); return this; } - private void callLocationService(Runnable runnable) { - if (Countly.instance().location() != null) { - runnable.run(); + private void callLocationService(Consumer call) { + SDKCore core = SDKCore.instance; + ModuleLocation location = core == null ? null : core.runningModule(CoreFeature.Location, ModuleLocation.class); + if (location != null) { + call.accept(location); } } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsInternalLimits.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsInternalLimits.java new file mode 100644 index 00000000..010027f7 --- /dev/null +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsInternalLimits.java @@ -0,0 +1,307 @@ +package ly.count.sdk.java.internal; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Applies the SDK internal limits of the SDK behavior settings: the key length {@code lkl}, the value + * size {@code lvs}, the number of segmentation entries {@code lsv}, the stack trace lines per thread + * {@code ltlpt} and the stack trace line length {@code ltl}. {@link Integer#MAX_VALUE} is no limit, + * and whatever no limit cuts is returned as the same instance. + *

+ * A map is never changed in place. One that has to change is rebuilt in sorted key order, so the + * outcome never depends on the order of the map: when two keys are cut to the same key the one that + * sorts last keeps its value, and over the entry limit the entries whose keys sort first are kept. + * Strings are cut in UTF-16 characters and never inside a surrogate pair. Every cut is logged as a + * warning that names the caller. + */ +final class UtilsInternalLimits { + + /** + * The order every limit walks keys in, a {@code null} key first. + */ + private static final Comparator KEY_ORDER = Comparator.nullsFirst(Comparator.naturalOrder()); + + /** + * Not instantiated: the limits are static helpers. + */ + private UtilsInternalLimits() { + } + + /** + * Cuts a key to the key length limit, {@code lkl}: an event key, a view name, a segmentation key + * or a custom user property key. + * + * @param key the key, {@code null} for none + * @param maxKeyLength the limit + * @param L logger + * @param tag the caller, for the log + * @return the key, cut when it is longer than the limit + */ + @Nullable + static String truncateKey(@Nullable String key, int maxKeyLength, @Nonnull Log L, @Nonnull String tag) { + if (key == null || key.length() <= maxKeyLength) { + return key; + } + + String truncated = truncate(key, maxKeyLength); + L.w(tag + ", the key [" + key + "] is longer than the key length limit of [" + maxKeyLength + "] of the SDK behavior settings, it is truncated to [" + truncated + "]"); + return truncated; + } + + /** + * Cuts a string value to the value size limit, {@code lvs}. + * + * @param value the value, {@code null} for none + * @param maxValueSize the limit + * @param L logger + * @param tag the caller, for the log + * @return the value, cut when it is longer than the limit + */ + @Nullable + static String truncateValue(@Nullable String value, int maxValueSize, @Nonnull Log L, @Nonnull String tag) { + if (value == null || value.length() <= maxValueSize) { + return value; + } + + String truncated = truncate(value, maxValueSize); + L.w(tag + ", a value of [" + value.length() + "] characters is longer than the value size limit of [" + maxValueSize + "] of the SDK behavior settings, it is truncated to [" + truncated + "]"); + return truncated; + } + + /** + * Cuts a value to the value size limit, {@code lvs}, when it is a string. A value of any other + * type is returned as it is, strings inside arrays included. + * + * @param value the value, {@code null} for none + * @param maxValueSize the limit + * @param L logger + * @param tag the caller, for the log + * @return the value, cut when it is a string longer than the limit + */ + @Nullable + static Object truncateIfString(@Nullable Object value, int maxValueSize, @Nonnull Log L, @Nonnull String tag) { + if (value instanceof String) { + return truncateValue((String) value, maxValueSize, L, tag); + } + return value; + } + + /** + * Applies the key length, value size and segmentation entry limits, {@code lkl}, {@code lvs} and + * {@code lsv}, to a segmentation. Keys and values are cut first, so two keys that end up the same + * are counted once. + * + * @param segmentation the segmentation, {@code null} for none, never changed + * @param limits the settings in effect + * @param L logger + * @param tag the caller, for the log + * @return the segmentation within the limits, the same instance when it already was + */ + @Nullable + static Map applySegmentationLimits(@Nullable Map segmentation, @Nonnull ConfigurationProvider limits, @Nonnull Log L, @Nonnull String tag) { + Map truncated = truncateSegmentationKeysAndValues(segmentation, limits.getMaxKeyLength(), limits.getMaxValueSize(), L, tag); + return limitSegmentationEntries(truncated, limits.getMaxSegmentationValues(), L, tag); + } + + /** + * Cuts the keys of a segmentation to the key length limit, {@code lkl}, and its string values to + * the value size limit, {@code lvs}. When two keys are cut to the same key, the entry whose key + * sorts last keeps its value. + * + * @param segmentation the segmentation, {@code null} for none, never changed + * @param maxKeyLength the key length limit + * @param maxValueSize the value size limit + * @param L logger + * @param tag the caller, for the log + * @return a copy in sorted key order when a key or a value was cut, otherwise the same instance + */ + @Nullable + static Map truncateSegmentationKeysAndValues(@Nullable Map segmentation, int maxKeyLength, int maxValueSize, @Nonnull Log L, @Nonnull String tag) { + if (segmentation == null || !exceedsKeyOrValueLimit(segmentation, maxKeyLength, maxValueSize)) { + return segmentation; + } + + Map truncated = new LinkedHashMap<>(); + for (String key : sortedKeys(segmentation.keySet())) { + String truncatedKey = truncateKey(key, maxKeyLength, L, tag); + if (truncated.containsKey(truncatedKey)) { + L.w(tag + ", the key [" + key + "] is truncated to the key of another segmentation entry, its value replaces that of [" + truncatedKey + "]"); + } + truncated.put(truncatedKey, truncateIfString(segmentation.get(key), maxValueSize, L, tag)); + } + return truncated; + } + + /** + * Keeps a segmentation within the segmentation entry limit, {@code lsv}, by keeping the entries + * whose keys sort first. + * + * @param segmentation the segmentation, {@code null} for none, never changed + * @param maxSegmentationValues the limit + * @param L logger + * @param tag the caller, for the log + * @return a copy in sorted key order when entries were dropped, otherwise the same instance + */ + @Nullable + static Map limitSegmentationEntries(@Nullable Map segmentation, int maxSegmentationValues, @Nonnull Log L, @Nonnull String tag) { + if (segmentation == null || segmentation.size() <= maxSegmentationValues) { + return segmentation; + } + + List keys = sortedKeys(segmentation.keySet()); + Map kept = new LinkedHashMap<>(); + for (String key : keys.subList(0, maxSegmentationValues)) { + kept.put(key, segmentation.get(key)); + } + + L.w(tag + ", the segmentation has [" + keys.size() + "] entries, over the limit of [" + maxSegmentationValues + "] of the SDK behavior settings, dropped the entries of the keys that sort last: " + keys.subList(maxSegmentationValues, keys.size())); + return kept; + } + + /** + * Cuts the string values of a map to the value size limit, {@code lvs}, leaving its keys and its + * number of entries as they are. + * + * @param map the map, never changed + * @param maxValueSize the limit + * @param L logger + * @param tag the caller, for the log + * @return a copy in the order of the map when a value was cut, otherwise the same instance + */ + @Nonnull + static Map truncateStringValues(@Nonnull Map map, int maxValueSize, @Nonnull Log L, @Nonnull String tag) { + if (!exceedsKeyOrValueLimit(map, Integer.MAX_VALUE, maxValueSize)) { + return map; + } + + Map truncated = new LinkedHashMap<>(); + for (Map.Entry entry : map.entrySet()) { + truncated.put(entry.getKey(), truncateIfString(entry.getValue(), maxValueSize, L, tag)); + } + return truncated; + } + + /** + * Cuts every line of a stack trace to the stack trace line length limit, {@code ltl}. Line breaks, + * {@code \n} or {@code \r\n}, are kept as they are and do not count as characters of the line. + * + * @param stackTrace the stack trace + * @param maxLineLength the limit + * @param L logger + * @param tag the caller, for the log + * @return the stack trace, a new one when a line was cut + */ + @Nonnull + static String truncateStackTraceLines(@Nonnull String stackTrace, int maxLineLength, @Nonnull Log L, @Nonnull String tag) { + StringBuilder truncated = null; + int truncatedLines = 0; + int lineStart = 0; + while (lineStart < stackTrace.length()) { + int lineBreak = stackTrace.indexOf('\n', lineStart); + int nextLineStart = lineBreak < 0 ? stackTrace.length() : lineBreak + 1; + int lineEnd = lineBreak < 0 ? stackTrace.length() : lineBreak; + if (lineEnd > lineStart && stackTrace.charAt(lineEnd - 1) == '\r') { + lineEnd--; + } + + if (lineEnd - lineStart > maxLineLength) { + if (truncated == null) { + truncated = new StringBuilder(stackTrace.length()); + truncated.append(stackTrace, 0, lineStart); + } + truncated.append(truncate(stackTrace.substring(lineStart, lineEnd), maxLineLength)).append(stackTrace, lineEnd, nextLineStart); + truncatedLines++; + } else if (truncated != null) { + truncated.append(stackTrace, lineStart, nextLineStart); + } + lineStart = nextLineStart; + } + + if (truncated == null) { + return stackTrace; + } + + L.w(tag + ", [" + truncatedLines + "] stack trace lines are longer than the line length limit of [" + maxLineLength + "] of the SDK behavior settings, they are truncated"); + return truncated.toString(); + } + + /** + * How many lines of the stack trace of one thread the stack trace lines per thread limit, + * {@code ltlpt}, keeps: the top ones. + * + * @param lineCount the number of lines of the stack trace + * @param maxLinesPerThread the limit + * @param threadName the thread, for the log + * @param L logger + * @param tag the caller, for the log + * @return the number of top lines to keep + */ + static int stackTraceLinesToKeep(int lineCount, int maxLinesPerThread, @Nullable String threadName, @Nonnull Log L, @Nonnull String tag) { + if (lineCount <= maxLinesPerThread) { + return lineCount; + } + + L.w(tag + ", the stack trace of the thread [" + threadName + "] has [" + lineCount + "] lines, over the limit of [" + maxLinesPerThread + "] lines per thread of the SDK behavior settings, dropped the bottom ones"); + return maxLinesPerThread; + } + + /** + * Whether a key of a map is longer than a key length limit or one of its string values is longer + * than a value size limit. + * + * @param map the map + * @param maxKeyLength the key length limit + * @param maxValueSize the value size limit + * @return {@code true} when a key or a value has to be cut + */ + private static boolean exceedsKeyOrValueLimit(@Nonnull Map map, int maxKeyLength, int maxValueSize) { + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + if (key != null && key.length() > maxKeyLength) { + return true; + } + Object value = entry.getValue(); + if (value instanceof String && ((String) value).length() > maxValueSize) { + return true; + } + } + return false; + } + + /** + * The keys of a map in the order every limit walks them. + * + * @param keys the keys + * @return the keys, sorted + */ + @Nonnull + private static List sortedKeys(@Nonnull Collection keys) { + List sorted = new ArrayList<>(keys); + sorted.sort(KEY_ORDER); + return sorted; + } + + /** + * Cuts a string that is longer than a limit. A cut never splits a surrogate pair: it falls before + * the pair, or after it when the pair starts the string, which is then one character over the limit. + * + * @param value the string, longer than the limit + * @param limit the limit, at least 1 + * @return the start of the string + */ + @Nonnull + private static String truncate(@Nonnull String value, int limit) { + int end = limit; + if (Character.isHighSurrogate(value.charAt(end - 1)) && Character.isLowSurrogate(value.charAt(end))) { + end = end == 1 ? 2 : end - 1; + } + return value.substring(0, end); + } +} diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsListingFilters.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsListingFilters.java new file mode 100644 index 00000000..2d62ab45 --- /dev/null +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/UtilsListingFilters.java @@ -0,0 +1,122 @@ +package ly.count.sdk.java.internal; + +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Applies the listing filters of the SDK behavior settings: the event filter, the user property + * filter, the segmentation filter and the segmentation filter of each event. A filter without names + * allows everything; otherwise a whitelist keeps only its names and a blacklist removes its names. + */ +final class UtilsListingFilters { + + /** + * Not instantiated: the filters are static helpers. + */ + private UtilsListingFilters() { + } + + /** + * Whether the event filter, {@code eb} or {@code ew}, lets a custom event be recorded. + * + * @param eventKey the key of the custom event + * @param configProvider the settings in effect + * @return {@code true} when the event may be recorded + */ + static boolean applyEventFilter(@Nonnull String eventKey, @Nonnull ConfigurationProvider configProvider) { + return applyListFilter(eventKey, configProvider.getEventFilterList()); + } + + /** + * Whether the user property filter, {@code upb} or {@code upw}, lets a custom user property be + * set or modified. + * + * @param propertyKey the key of the custom user property + * @param configProvider the settings in effect + * @return {@code true} when the property may be set or modified + */ + static boolean applyUserPropertyFilter(@Nullable String propertyKey, @Nonnull ConfigurationProvider configProvider) { + return applyListFilter(propertyKey, configProvider.getUserPropertyFilterList()); + } + + /** + * Removes from a segmentation the keys the segmentation filter, {@code sb} or {@code sw}, does not allow. + * + * @param segmentation the segmentation, changed in place + * @param configProvider the settings in effect + * @param L logger + */ + static void applySegmentationFilter(@Nonnull Map segmentation, @Nonnull ConfigurationProvider configProvider, @Nonnull Log L) { + if (segmentation.isEmpty()) { + return; + } + + ConfigurationProvider.FilterList> filter = configProvider.getSegmentationFilterList(); + applyMapFilter(segmentation, filter.getFilterList(), filter.isWhitelist(), L); + } + + /** + * Removes from the segmentation of an event the keys that the segmentation filter of that event, + * {@code esb} or {@code esw}, does not allow. An event without names in the filter keeps every key. + * + * @param eventKey the key of the event + * @param segmentation the segmentation, changed in place + * @param configProvider the settings in effect + * @param L logger + */ + static void applyEventSegmentationFilter(@Nonnull String eventKey, @Nonnull Map segmentation, @Nonnull ConfigurationProvider configProvider, @Nonnull Log L) { + ConfigurationProvider.FilterList>> filter = configProvider.getEventSegmentationFilterList(); + if (segmentation.isEmpty() || filter.getFilterList().isEmpty()) { + return; + } + + Set names = filter.getFilterList().get(eventKey); + if (names == null) { + return; + } + + applyMapFilter(segmentation, names, filter.isWhitelist(), L); + } + + /** + * Removes the keys of a map that a filter does not allow. + * + * @param map the map, changed in place + * @param names the names of the filter + * @param isWhitelist {@code true} when only the names are allowed, {@code false} when they are rejected + * @param L logger + */ + private static void applyMapFilter(@Nonnull Map map, @Nonnull Set names, boolean isWhitelist, @Nonnull Log L) { + if (names.isEmpty()) { + return; + } + + Iterator> entries = map.entrySet().iterator(); + while (entries.hasNext()) { + String key = entries.next().getKey(); + if (isWhitelist != names.contains(key)) { + entries.remove(); + L.d("[UtilsListingFilters] applyMapFilter, removed the segmentation key [" + key + "], " + (isWhitelist ? "it is not in the whitelist" : "it is in the blacklist")); + } + } + } + + /** + * Whether a filter allows a name. + * + * @param name the name + * @param filter the filter + * @return {@code true} when the filter has no names or allows this one + */ + private static boolean applyListFilter(@Nullable String name, @Nonnull ConfigurationProvider.FilterList> filter) { + Set names = filter.getFilterList(); + if (names.isEmpty()) { + return true; + } + + return filter.isWhitelist() == names.contains(name); + } +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/CoreRobustnessTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/CoreRobustnessTests.java index 860fb210..0bc887da 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/CoreRobustnessTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/CoreRobustnessTests.java @@ -1,7 +1,9 @@ package ly.count.sdk.java.internal; import java.io.File; +import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; import ly.count.sdk.java.Config; import ly.count.sdk.java.Countly; import org.junit.After; @@ -76,36 +78,42 @@ public void init_withANullConfig_isRefusedWithoutCrashing() { } /** - * Feature accessors for features that were never enabled return null rather than throwing. + * Feature accessors for features that were never enabled hand out an interface that ignores every + * call, rather than null or throwing. *

* They dereferenced a module that was never built, and consent is not the gate that stops them: * with consent not required, the consent check passes and the null module was used anyway. */ @Test - public void accessors_forFeaturesThatWereNeverEnabled_returnNull() { + public void accessors_forFeaturesThatWereNeverEnabled_handOutInterfacesThatIgnoreCalls() { Countly.instance().init(TestUtils.getConfigEvents(2)); - Assert.assertNull(Countly.instance().feedback()); - Assert.assertNull(Countly.instance().crashes()); - Assert.assertNull(Countly.instance().remoteConfig()); - Assert.assertNull(Countly.instance().content()); + Assert.assertSame(SDKCore.dormantFeedback, Countly.instance().feedback()); + Assert.assertSame(SDKCore.dormantCrashes, Countly.instance().crashes()); + Assert.assertSame(SDKCore.dormantRemoteConfig, Countly.instance().remoteConfig()); + Assert.assertSame(SDKCore.dormantContent, Countly.instance().content()); + int queued = TestUtils.getCurrentRQ().length; + Countly.instance().crashes().recordHandledException(new RuntimeException("never enabled")); + Countly.instance().content().enterContentZone(); + Assert.assertEquals(queued, TestUtils.getCurrentRQ().length); // The feature that IS enabled still works, so the guards did not break the happy path. - Assert.assertNotNull(Countly.instance().events()); + Assert.assertNotSame(SDKCore.dormantEvents, Countly.instance().events()); } /** - * Accessors used before init log and return null. "location" logged through a logger that does - * not exist yet, unlike every sibling accessor. + * Accessors used before init log and hand out an interface that ignores calls, backend mode + * aside, which still answers null. "location" logged through a logger that does not exist yet, + * unlike every sibling accessor. */ @Test - public void accessors_beforeInit_returnNullWithoutCrashing() { + public void accessors_beforeInit_doNotThrow() { Countly.instance().halt(); - Assert.assertNull(Countly.instance().location()); - Assert.assertNull(Countly.instance().feedback()); - Assert.assertNull(Countly.instance().content()); - Assert.assertNull(Countly.instance().remoteConfig()); + Assert.assertSame(SDKCore.dormantLocation, Countly.instance().location()); + Assert.assertSame(SDKCore.dormantFeedback, Countly.instance().feedback()); + Assert.assertSame(SDKCore.dormantContent, Countly.instance().content()); + Assert.assertSame(SDKCore.dormantRemoteConfig, Countly.instance().remoteConfig()); Assert.assertNull(Countly.instance().backendM()); } @@ -166,6 +174,62 @@ public void aStorableThatCannotSerialise_leavesNoFileBehind() { Assert.assertEquals("no file should have been created, found: " + litter.length, 0, litter.length); } + /** + * A fresh init reads storage files that do not exist yet, which is the normal first run state: + * each one is logged at debug level, and no error is logged for any of them. + */ + @Test + public void freshInit_logsTheStorageFilesThatDoNotExistYet_atDebugLevel() { + List missingFileLines = new CopyOnWriteArrayList<>(); + Countly.instance().init(TestUtils.getConfigEvents(2).setLoggingLevel(Config.LoggingLevel.VERBOSE) + .setLogListener((message, level) -> { + if (message.contains("No data for file")) { + missingFileLines.add(level + " " + message); + } + })); + + Assert.assertFalse(missingFileLines.isEmpty()); + for (String line : missingFileLines) { + Assert.assertTrue(line, line.startsWith(Config.LoggingLevel.DEBUG + " ")); + } + } + + /** + * A storage file that exists but cannot be opened, unlike a missing one, is logged as an error. + */ + @Test + public void init_withAStorageFileThatCannotBeOpened_logsAnError() { + Assert.assertTrue(new File(TestUtils.getTestSDirectory(), "[CLY]_timedEvent_0").mkdir()); + List errors = new CopyOnWriteArrayList<>(); + Countly.instance().init(TestUtils.getConfigEvents(2).setLoggingLevel(Config.LoggingLevel.VERBOSE) + .setLogListener((message, level) -> { + if (level == Config.LoggingLevel.ERROR && message.contains("timedEvent_0")) { + errors.add(message); + } + })); + + Assert.assertEquals(errors.toString(), 1, errors.size()); + Assert.assertTrue(errors.get(0), errors.get(0).contains("Error while opening file")); + } + + /** + * A second init is refused before it changes the running SDK, so given the same configuration + * object again it leaves the logger the running SDK and its configuration log through in place. + */ + @Test + public void secondInit_withTheSameConfigObject_keepsTheRunningLogger() { + InternalConfig config = new InternalConfig(TestUtils.getConfigEvents(2)); + Countly.instance().init(config); + Log running = SDKCore.instance.L; + Assert.assertSame(running, config.getLogger()); + + Countly.instance().init(config); + + Assert.assertTrue(Countly.isInitialized()); + Assert.assertSame(running, SDKCore.instance.L); + Assert.assertSame(running, config.getLogger()); + } + /** * The queued crash request, or null when none was queued. * diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/CountlyFacadeTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/CountlyFacadeTests.java index 973fdcca..be8f9dd2 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/CountlyFacadeTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/CountlyFacadeTests.java @@ -1,7 +1,12 @@ package ly.count.sdk.java.internal; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import ly.count.sdk.java.Config; import ly.count.sdk.java.Countly; @@ -18,9 +23,9 @@ /** * The static {@link Countly} facade and the {@link SDKCore} accessors behind it. *

- * Two contracts are covered: before {@code init} every accessor must answer with null and a logged - * error instead of throwing, and after {@code init} the deprecated {@link Usage} API must still route - * to the live session. Both matter because they are the surface an integration touches first. + * Two contracts are covered: before {@code init} no accessor may throw, and every feature interface + * is handed out but ignores calls, and after {@code init} the deprecated {@link Usage} API must still + * route to the live session. Both matter because they are the surface an integration touches first. */ @RunWith(JUnit4.class) public class CountlyFacadeTests { @@ -39,26 +44,41 @@ public void stop() { /** * Nothing may throw before {@code init}. Every module accessor and every state query is walked in - * one pass, so a newly added accessor that forgets its guard shows up here. + * one pass, so a newly added accessor that forgets its guard shows up here. Each feature interface + * is handed out and ignores calls, so code that calls it before init does not throw either, and a + * call with a callback gets its answer. */ @Test - public void beforeInit_everyAccessorAnswersNullInsteadOfThrowing() { + public void beforeInit_noAccessorThrows_andEveryFeatureInterfaceIgnoresCalls() { Assert.assertFalse(Countly.isInitialized()); Assert.assertNull(Countly.session()); Assert.assertNull(Countly.instance().backendM()); Assert.assertNull(Countly.backendMode()); - Assert.assertNull(Countly.instance().feedback()); - Assert.assertNull(Countly.instance().content()); - Assert.assertNull(Countly.instance().events()); - Assert.assertNull(Countly.instance().views()); - Assert.assertNull(Countly.instance().crashes()); - Assert.assertNull(Countly.instance().remoteConfig()); - Assert.assertNull(Countly.instance().deviceId()); - Assert.assertNull(Countly.instance().userProfile()); - // Countly.location() is deliberately NOT called here: its own uninitialised guard logs - // through the instance logger without a null check, so it throws NPE before it can return - // null. Reported as a bug rather than asserted, so a fix does not have to touch this test. + Assert.assertSame(SDKCore.dormantFeedback, Countly.instance().feedback()); + Assert.assertSame(SDKCore.dormantContent, Countly.instance().content()); + Assert.assertSame(SDKCore.dormantEvents, Countly.instance().events()); + Assert.assertSame(SDKCore.dormantViews, Countly.instance().views()); + Assert.assertSame(SDKCore.dormantCrashes, Countly.instance().crashes()); + Assert.assertSame(SDKCore.dormantRemoteConfig, Countly.instance().remoteConfig()); + Assert.assertSame(SDKCore.dormantDeviceId, Countly.instance().deviceId()); + Assert.assertSame(SDKCore.dormantUserProfile, Countly.instance().userProfile()); + Assert.assertSame(SDKCore.dormantLocation, Countly.instance().location()); + + Assert.assertFalse(Countly.instance().events().startEvent("before_init")); + Countly.instance().events().recordEvent("before_init"); + Assert.assertNull(Countly.instance().views().startView("home")); + Countly.instance().crashes().recordHandledException(new RuntimeException("before init")); + Assert.assertNull(Countly.instance().deviceId().getID()); + Assert.assertTrue(Countly.instance().remoteConfig().getValues().isEmpty()); + Countly.instance().userProfile().setProperty("plan", "pro"); + Countly.instance().location().setLocation("TR", "Izmir", "38.4,27.1", null); + Countly.instance().content().enterContentZone(); + List errors = new CopyOnWriteArrayList<>(); + Countly.instance().feedback().getAvailableFeedbackWidgets((widgets, error) -> errors.add(error)); + Countly.instance().remoteConfig().downloadAllKeys((result, error, fullValueUpdate, values) -> errors.add(error)); + Assert.assertEquals(2, errors.size()); + Assert.assertFalse(Countly.isInitialized()); // isTracking short circuits on the uninitialised check rather than dereferencing the SDK. for (Config.Feature feature : Config.Feature.values()) { @@ -258,11 +278,12 @@ public void stop(InternalConfig config, boolean clear) { } /** - * Consent gating of the accessors. With consent required and nothing granted the feature - * interfaces must be withheld; granting afterwards must hand them over. + * Consent gating of the accessors. With consent required and nothing granted every feature + * interface ignores calls; granting afterwards hands over the running ones, and removing the + * consent again hands out the ignoring ones. */ @Test - public void consentGating_withholdsThenReleasesTheFeatureInterfaces() { + public void consentGating_handsOutInterfacesThatIgnoreCalls_untilTheConsentIsGiven() { Config config = TestUtils.getBaseConfig() .enableFeatures(Config.Feature.Events, Config.Feature.Views, Config.Feature.Location, Config.Feature.Feedback, Config.Feature.RemoteConfig, Config.Feature.CrashReporting) @@ -270,28 +291,265 @@ public void consentGating_withholdsThenReleasesTheFeatureInterfaces() { Countly.instance().init(config); Assert.assertFalse(Countly.isTracking(Config.Feature.Events)); - Assert.assertNull(Countly.instance().events()); - Assert.assertNull(Countly.instance().views()); - Assert.assertNull(Countly.instance().location()); - Assert.assertNull(Countly.instance().feedback()); - Assert.assertNull(Countly.instance().remoteConfig()); - Assert.assertNull(Countly.instance().crashes()); + Assert.assertSame(SDKCore.dormantEvents, Countly.instance().events()); + Assert.assertSame(SDKCore.dormantViews, Countly.instance().views()); + Assert.assertSame(SDKCore.dormantLocation, Countly.instance().location()); + Assert.assertSame(SDKCore.dormantFeedback, Countly.instance().feedback()); + Assert.assertSame(SDKCore.dormantRemoteConfig, Countly.instance().remoteConfig()); + Assert.assertSame(SDKCore.dormantCrashes, Countly.instance().crashes()); + Countly.instance().events().recordEvent("before_consent"); + TestUtils.validateEQSize(0); Countly.onConsent(Config.Feature.values()); Assert.assertTrue(Countly.isTracking(Config.Feature.Events)); - Assert.assertNotNull(Countly.instance().events()); - Assert.assertNotNull(Countly.instance().views()); - Assert.assertNotNull(Countly.instance().location()); - Assert.assertNotNull(Countly.instance().feedback()); - Assert.assertNotNull(Countly.instance().remoteConfig()); - Assert.assertNotNull(Countly.instance().crashes()); + Assert.assertSame(SDKCore.instance.module(ModuleEvents.class).eventsInterface, Countly.instance().events()); + Assert.assertNotSame(SDKCore.dormantViews, Countly.instance().views()); + Assert.assertNotSame(SDKCore.dormantLocation, Countly.instance().location()); + Assert.assertNotSame(SDKCore.dormantFeedback, Countly.instance().feedback()); + Assert.assertNotSame(SDKCore.dormantRemoteConfig, Countly.instance().remoteConfig()); + Assert.assertNotSame(SDKCore.dormantCrashes, Countly.instance().crashes()); Countly.onConsentRemoval(Config.Feature.values()); Assert.assertFalse(Countly.isTracking(Config.Feature.Events)); - Assert.assertNull(Countly.instance().events()); - Assert.assertNull(Countly.instance().views()); + Assert.assertSame(SDKCore.dormantEvents, Countly.instance().events()); + Assert.assertSame(SDKCore.dormantViews, Countly.instance().views()); + } + + /** + * Every interface sends its calls to the module of its feature once that runs, wherever it came + * from, as a caller may keep the one it got first. + *

+ * Verifies that an events interface handed out before init and one handed out before the consent + * record once the consent is given, that a views interface handed out before the consent starts a + * view then, and that a running events interface kept across a removal of its consent ignores + * calls until the consent is given again and then records into the new module. + */ + @Test + public void aKeptInterface_reachesTheModuleOnceItsFeatureRuns() { + ModuleEvents.Events beforeInit = Countly.instance().events(); + Countly.instance().init(TestUtils.getBaseConfig().enableFeatures(Config.Feature.Events, Config.Feature.Views) + .setRequiresConsent(true).setEventQueueSizeToSend(100)); + ModuleEvents.Events beforeConsent = Countly.instance().events(); + ModuleViews.Views viewsBeforeConsent = Countly.instance().views(); + beforeConsent.recordEvent("ignored_before_consent"); + Assert.assertNull(viewsBeforeConsent.startView("ignored")); + TestUtils.validateEQSize(0); + + Countly.onConsent(Config.Feature.Events, Config.Feature.Views); + + beforeInit.recordEvent("from_before_init"); + beforeConsent.recordEvent("from_before_consent"); + Assert.assertNotNull(viewsBeforeConsent.startView("home")); + Assert.assertTrue(beforeConsent.startEvent("timed")); + Assert.assertTrue(Countly.instance().events().endEvent("timed")); + Assert.assertEquals(Arrays.asList("from_before_init", "from_before_consent", ModuleViews.KEY_VIEW_EVENT, "timed"), TestUtils.keysInEQ()); + + ModuleEvents.Events running = Countly.instance().events(); + Countly.onConsentRemoval(Config.Feature.Events); + running.recordEvent("ignored_after_removal"); + TestUtils.validateEQSize(0); + + Countly.onConsent(Config.Feature.Events); + running.recordEvent("after_consent_again"); + Assert.assertEquals(Collections.singletonList("after_consent_again"), TestUtils.keysInEQ()); + } + + /** + * Events carry the view IDs of the views module that runs now. + *

+ * Verifies that while the views consent is removed an event carries no current view ID, and that + * once it is given again an event carries the ID of the view the new module started, as does the + * next view as its previous one. + */ + @Test + public void viewIds_followTheViewsModuleThatRuns_acrossAViewsConsentChange() { + Countly.instance().init(TestUtils.getBaseConfig().enableFeatures(Config.Feature.Events, Config.Feature.Views) + .setRequiresConsent(true).setEventQueueSizeToSend(100)); + Countly.onConsent(Config.Feature.Events, Config.Feature.Views); + Assert.assertNotNull(Countly.instance().views().startView("first")); + + Countly.onConsentRemoval(Config.Feature.Views); + Countly.instance().events().recordEvent("without_views"); + Countly.onConsent(Config.Feature.Views); + String second = Countly.instance().views().startView("second"); + Countly.instance().events().recordEvent("with_views"); + Countly.instance().views().startView("third"); + + List queued = TestUtils.getCurrentEQ(); + Assert.assertEquals(Arrays.asList(ModuleViews.KEY_VIEW_EVENT, "without_views", ModuleViews.KEY_VIEW_EVENT, "with_views", ModuleViews.KEY_VIEW_EVENT), TestUtils.keysInEQ()); + Assert.assertTrue(Utils.isEmptyOrNull(queued.get(1).cvid)); + Assert.assertEquals(second, queued.get(3).cvid); + Assert.assertEquals(second, queued.get(4).pvid); + } + + /** + * A module that fails on a timer tick is logged and the timer keeps ticking for every module. + */ + @Test + public void aModuleFailingOnATick_doesNotStopTheTimer() throws InterruptedException { + AtomicInteger ticks = new AtomicInteger(); + SDKCore.testDummyModule = new ModuleBase() { + /** + * Counts the tick and fails on the first one. + */ + @Override + public void onTimer() { + if (ticks.incrementAndGet() == 1) { + throw new IllegalStateException("failing on the first tick"); + } + } + }; + try { + Countly.instance().init(TestUtils.getBaseConfig().setUpdateSessionTimerDelay(1)); + long deadline = System.currentTimeMillis() + 10_000; + while (ticks.get() < 2 && System.currentTimeMillis() < deadline) { + Thread.sleep(50); + } + Assert.assertTrue("ticks: " + ticks.get(), ticks.get() >= 2); + } finally { + SDKCore.testDummyModule = null; + } + } + + /** + * The deprecated API and the user editor log no user data before the feature it belongs to + * checks its consent, as such a line can be gathered for the server under another consent. + */ + @Test + public void deprecatedCallsAndTheUserEditor_logNoUserData_beforeTheirConsentCheck() { + List lines = new CopyOnWriteArrayList<>(); + Countly.instance().init(TestUtils.getBaseConfig().setRequiresConsent(true).setLoggingLevel(Config.LoggingLevel.VERBOSE) + .enableFeatures(Config.Feature.Sessions, Config.Feature.Location, Config.Feature.Views, Config.Feature.CrashReporting, Config.Feature.UserProfiles) + .setLogListener((message, level) -> { + if (message.startsWith("[Countly]") || message.startsWith("[UserEditorImpl]")) { + lines.add(message); + } + })); + Countly.onConsent(Config.Feature.Sessions); + Map segments = new HashMap<>(); + segments.put("secret_key", "secret_value"); + + Countly.api().addLocation(12.345_678, 98.765_432); + Countly.api().view("SecretViewName"); + Countly.api().addCrashReport(new RuntimeException("secret crash message"), false, "secret_crash_name", segments); + Countly.api().addParam("param_key", "secret_param_value"); + Countly.instance().user().edit().setLocale("tr_TR").setCountry("TR_country").setCity("SecretCity") + .setLocation(1.234_567, 2.345_678).setLocation("38.456789,27.123456").setLocation("not a location").commit(); + + Assert.assertFalse(lines.isEmpty()); + for (String line : lines) { + for (String secret : Arrays.asList("12.345678", "SecretViewName", "secret", "tr_TR", "TR_country", "SecretCity", "1.234567", "38.456789", "not a location")) { + Assert.assertFalse(line, line.contains(secret)); + } + } + } + + /** + * Removing a consent stops the modules of the features it names and no others, so the device ID + * module, which has no consent of its own, keeps running. + *

+ * Verifies that after the location consent is removed the location interface ignores calls while + * the device ID, events, views and user profile interfaces keep working and recording, and that + * an events interface kept from before the events consent is removed ignores calls instead of + * recording into the stopped module. + */ + @Test + public void consentRemoval_stopsOnlyTheFeaturesItNames_andAKeptInterfaceIgnoresCalls() { + Countly.instance().init(TestUtils.getBaseConfig() + .enableFeatures(Config.Feature.Events, Config.Feature.Views, Config.Feature.Location, Config.Feature.UserProfiles) + .setRequiresConsent(true).setEventQueueSizeToSend(100)); + Countly.onConsent(Config.Feature.values()); + ModuleEvents.Events keptEvents = Countly.instance().events(); + + Countly.onConsentRemoval(Config.Feature.Location); + + Assert.assertFalse(Countly.isTracking(Config.Feature.Location)); + Assert.assertSame(SDKCore.dormantLocation, Countly.instance().location()); + Assert.assertEquals(TestUtils.DEVICE_ID, Countly.instance().deviceId().getID()); + Assert.assertSame(keptEvents, Countly.instance().events()); + keptEvents.recordEvent("after_location_removal"); + Countly.instance().views().startView("home"); + Assert.assertEquals(2, TestUtils.getCurrentEQ().size()); + Countly.instance().userProfile().setProperty("plan", "pro"); + Countly.instance().userProfile().save(); + Assert.assertEquals(1, requestsContaining("user_details")); + + Countly.onConsentRemoval(Config.Feature.Events); + + Assert.assertSame(SDKCore.dormantEvents, Countly.instance().events()); + keptEvents.recordEvent("after_events_removal"); + Assert.assertFalse(keptEvents.startEvent("timed_after_events_removal")); + TestUtils.validateEQSize(0); + Assert.assertEquals(TestUtils.DEVICE_ID, Countly.instance().deviceId().getID()); + } + + /** + * Consent changes add and remove modules while the transport hands responses to their owners, + * and every response still reaches its owner. + *

+ * Verifies that while another thread keeps giving and removing a consent, handing responses + * over throws nothing and each one reaches the module that owns the request, that the consent + * changes throw nothing either, and that the last removal took effect. + */ + @Test + public void requestCompletions_reachTheirOwner_whileConsentChangesAddAndRemoveModules() throws InterruptedException { + AtomicInteger delivered = new AtomicInteger(); + SDKCore.testDummyModule = new ModuleBase() { + /** + * Counts a response handed over. + * + * @param request the request that was sent + * @param response the response body + * @param responseCode the response code + */ + @Override + public void onRequestCompleted(Request request, String response, int responseCode) { + delivered.incrementAndGet(); + } + }; + try { + Countly.instance().init(TestUtils.getBaseConfig().enableFeatures(Config.Feature.Feedback).setRequiresConsent(true)); + SDKCore core = SDKCore.instance; + Class owner = SDKCore.testDummyModule.getClass(); + Request request = ModuleRequests.nonSessionRequest(core.config); + List failures = new CopyOnWriteArrayList<>(); + AtomicInteger handedOver = new AtomicInteger(); + AtomicBoolean changingConsents = new AtomicBoolean(true); + + Thread completions = new Thread(() -> { + while (changingConsents.get()) { + try { + core.onRequestCompleted(request, "{\"result\":\"Success\"}", 200, owner); + handedOver.incrementAndGet(); + } catch (Throwable t) { + failures.add(t); + return; + } + } + }); + completions.start(); + try { + for (int i = 0; i < 2000 && failures.isEmpty(); i++) { + Countly.onConsent(Config.Feature.Feedback); + Countly.onConsentRemoval(Config.Feature.Feedback); + } + } catch (RuntimeException e) { + failures.add(e); + } finally { + changingConsents.set(false); + completions.join(30_000); + } + + Assert.assertFalse(completions.isAlive()); + Assert.assertTrue("failed: " + failures, failures.isEmpty()); + Assert.assertTrue(handedOver.get() > 0); + Assert.assertEquals(handedOver.get(), delivered.get()); + Assert.assertSame(SDKCore.dormantFeedback, Countly.instance().feedback()); + } finally { + SDKCore.testDummyModule = null; + } } // endregion diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/CrashFlowTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/CrashFlowTests.java index 2b488c34..7a0f0cc6 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/CrashFlowTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/CrashFlowTests.java @@ -3,6 +3,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; import ly.count.sdk.java.Config; import ly.count.sdk.java.Countly; import ly.count.sdk.java.Crash; @@ -138,8 +139,36 @@ public void crashReporting_refusesBadInputAndStandsDownUnderBackendMode() { } /** - * Consent gating. With consent required and CrashReporting not granted the interface is withheld - * entirely; once granted the crash flows. + * With consent required, an unhandled crash is recorded once the crash reporting consent is given + * after init, and still reaches the handler the application installed. + */ + @Test + public void unhandledCrash_isRecorded_whenTheConsentIsGivenAfterInit() throws InterruptedException { + Thread.UncaughtExceptionHandler before = Thread.getDefaultUncaughtExceptionHandler(); + List reachedTheApp = new CopyOnWriteArrayList<>(); + Thread.setDefaultUncaughtExceptionHandler((thread, throwable) -> reachedTheApp.add(throwable)); + try { + Countly.instance().init(TestUtils.getBaseConfig().enableFeatures(Config.Feature.CrashReporting).setRequiresConsent(true)); + Countly.onConsent(Config.Feature.CrashReporting); + + Thread crashing = new Thread(() -> { + throw new IllegalStateException("unhandled after consent"); + }); + crashing.start(); + crashing.join(); + Storage.await(SDKCore.instance.config.getLogger()); + + Assert.assertEquals(1, reachedTheApp.size()); + Assert.assertEquals(1, crashRequestCount()); + } finally { + Countly.instance().halt(); + Thread.setDefaultUncaughtExceptionHandler(before); + } + } + + /** + * Consent gating. With consent required and CrashReporting not granted the interface ignores every + * call; once granted the crash flows. */ @Test public void crashReporting_isGatedOnConsent() { @@ -148,7 +177,8 @@ public void crashReporting_isGatedOnConsent() { .setRequiresConsent(true); Countly.instance().init(config); - Assert.assertNull("no consent means no crash interface", Countly.instance().crashes()); + Assert.assertSame("no consent means a crash interface that ignores calls", SDKCore.dormantCrashes, Countly.instance().crashes()); + Countly.instance().crashes().recordHandledException(new RuntimeException("before consent")); Countly.onConsent(Config.Feature.CrashReporting); Assert.assertNotNull(Countly.instance().crashes()); diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/DefaultNetworkingRecheckTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/DefaultNetworkingRecheckTests.java new file mode 100644 index 00000000..c6c071df --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/DefaultNetworkingRecheckTests.java @@ -0,0 +1,79 @@ +package ly.count.sdk.java.internal; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import ly.count.sdk.java.Countly; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static org.mockito.Mockito.mock; + +/** + * The checks {@link DefaultNetworking} keeps while it drains the request queue. + */ +@RunWith(JUnit4.class) +public class DefaultNetworkingRecheckTests { + + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + } + + @After + public void afterTest() { + Countly.instance().halt(); + } + + /** + * A drain that fails still runs the check asked for while it ran, so a request queued meanwhile + * does not wait for the next check. + */ + @Test + public void aFailingDrain_stillRunsTheCheckAskedForMeanwhile() throws Exception { + Countly.instance().init(TestUtils.getBaseConfig()); + InternalConfig config = SDKCore.instance.config; + CountDownLatch firstDrainRuns = new CountDownLatch(1); + CountDownLatch checkAskedFor = new CountDownLatch(1); + CountDownLatch nextDrainRan = new CountDownLatch(1); + AtomicInteger drains = new AtomicInteger(); + DefaultNetworking networking = new DefaultNetworking() { + /** + * Fails the first drain once a check was asked for while it ran, and records the next. + */ + @Override + protected Tasks.Task submit(InternalConfig config, boolean followsOwnTransportWork) { + return new Tasks.Task(Tasks.ID_STRICT) { + /** + * Fails the first time, after the check was asked for, and records the next. + */ + @Override + public Boolean call() throws Exception { + if (drains.incrementAndGet() == 1) { + firstDrainRuns.countDown(); + Assert.assertTrue(checkAskedFor.await(10, TimeUnit.SECONDS)); + throw new IllegalStateException("the first drain fails"); + } + nextDrainRan.countDown(); + return false; + } + }; + } + }; + networking.init(config, mock(IStorageForRequestQueue.class)); + try { + networking.check(config); + Assert.assertTrue(firstDrainRuns.await(10, TimeUnit.SECONDS)); + networking.check(config); + checkAskedFor.countDown(); + + Assert.assertTrue("the check asked for during the failing drain never ran", nextDrainRan.await(10, TimeUnit.SECONDS)); + } finally { + networking.stop(config); + } + } +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/JsonFileStorageTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/JsonFileStorageTests.java index 8111d5dc..09fccd1f 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/JsonFileStorageTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/JsonFileStorageTests.java @@ -4,6 +4,9 @@ import java.io.IOException; import java.math.BigDecimal; import java.nio.file.Files; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import org.json.JSONObject; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -238,6 +241,168 @@ public void deleteAndSave() { validateStorageSize(0, 1); } + /** + * "getJsonObj" hands out a deep copy. + *

+ * Verifies that changing the returned object, and an object and an array nested in it, changes + * neither what the storage returns next nor what "save" writes; that a missing key and a value + * that is not an object return the default; that the changed copy is stored once it is handed back + * through "addAndSave"; and that saving on one thread while another changes its copy and stores it, + * as the remote config values are, never fails to serialize and leaves every key on disk. + */ + @Test + public void getJsonObj_returnsADeepCopy_soAChangeReachesTheStorageOnlyThroughAdd() throws InterruptedException { + Log logger = Mockito.mock(Log.class); + setupJsonFile("{\"rc\":{\"color\":{\"v\":\"red\",\"c\":1},\"list\":[{\"a\":1}]},\"plain\":\"text\"}"); + storage = new JsonFileStorage(jsonFile(), logger); + + JSONObject copy = storage.getJsonObj("rc", null); + copy.getJSONObject("color").put("c", 0); + copy.getJSONArray("list").getJSONObject(0).put("a", 2); + copy.put("size", "large"); + + JSONObject stored = storage.getJsonObj("rc", null); + Assert.assertNotSame(copy, stored); + Assert.assertEquals(1, stored.getJSONObject("color").getInt("c")); + Assert.assertEquals(1, stored.getJSONArray("list").getJSONObject(0).getInt("a")); + Assert.assertFalse(stored.has("size")); + storage.save(); + Assert.assertTrue(stored.similar(TestUtils.readJsonFile(jsonFile()).getJSONObject("rc"))); + + JSONObject fallback = new JSONObject(); + Assert.assertSame(fallback, storage.getJsonObj("missing", fallback)); + Assert.assertSame(fallback, storage.getJsonObj("plain", fallback)); + Assert.assertNull(storage.getJsonObj("missing", null)); + + storage.addAndSave("rc", copy); + Assert.assertTrue(copy.similar(storage.getJsonObj("rc", null))); + Assert.assertTrue(copy.similar(TestUtils.readJsonFile(jsonFile()).getJSONObject("rc"))); + + storage.add("device", "device_id"); + List failures = new CopyOnWriteArrayList<>(); + long deadline = System.currentTimeMillis() + 400; + Thread changer = new Thread(() -> { + try { + for (int round = 0; System.currentTimeMillis() < deadline; round++) { + JSONObject values = storage.getJsonObj("rc", new JSONObject()); + for (int i = 0; i < 50; i++) { + values.put("key_" + ((round * 50 + i) % 400), new JSONObject().put("v", i).put("c", 1)); + } + for (int i = 0; i < 20; i++) { + values.remove("key_" + ((round * 37 + i) % 400)); + } + storage.add("rc", values); + } + } catch (Throwable t) { + failures.add(t); + } + }); + changer.start(); + try { + while (System.currentTimeMillis() < deadline) { + storage.save(); + } + } catch (Throwable t) { + failures.add(t); + } + changer.join(); + + Assert.assertTrue("failures: " + failures, failures.isEmpty()); + Mockito.verify(logger, Mockito.never()).e(Mockito.startsWith("[JsonFileStorage] save, Failed to serialize")); + storage.save(); + JSONObject onDisk = TestUtils.readJsonFile(jsonFile()); + Assert.assertEquals("device_id", onDisk.getString("device")); + Assert.assertEquals("text", onDisk.getString("plain")); + Assert.assertTrue(onDisk.getJSONObject("rc").similar(storage.getJsonObj("rc", null))); + } + + /** + * "save" when the data cannot be serialized. + *

+ * Verifies that the file keeps what it had instead of being emptied, and that the failure is logged. + */ + @Test + public void save_leavesTheFileAsItIs_whenTheDataCannotBeSerialized() { + Log logger = Mockito.mock(Log.class); + setupJsonFile("{\"device\":\"device_id\"}"); + storage = new JsonFileStorage(jsonFile(), logger); + storage.add("broken", new JSONObject() { + /** + * Fails as a value that cannot be serialized does. + * + * @param writer the writer + * @param indentFactor the indentation of each level + * @param indent the indentation of this level + * @return never + */ + @Override + public java.io.Writer write(java.io.Writer writer, int indentFactor, int indent) { + throw new IllegalStateException("cannot be written"); + } + }); + + storage.save(); + + Assert.assertEquals("device_id", TestUtils.readJsonFile(jsonFile()).getString("device")); + Assert.assertFalse(TestUtils.readJsonFile(jsonFile()).has("broken")); + Mockito.verify(logger).e(Mockito.startsWith("[JsonFileStorage] save, Failed to serialize the data")); + } + + /** + * A json file cut short, as a process stopped while writing leaves it, does not stop the storage. + *

+ * Verifies that the storage starts empty, that the failure is logged, and that the next save + * replaces the broken file with valid json. + */ + @Test + public void readJsonFile_brokenJson_startsEmptyAndTheNextSaveReplacesIt() { + Log logger = Mockito.mock(Log.class); + setupJsonFile("{\"did\":\"generated_uuid\",\"sc\":\"{\\\"c\\\":{"); + storage = new JsonFileStorage(jsonFile(), logger); + + Assert.assertEquals(0, storage.size()); + Mockito.verify(logger).e(Mockito.startsWith("[JsonFileStorage] readJsonFile, The json file is not valid json")); + + storage.addAndSave("key", "value"); + Assert.assertEquals("value", TestUtils.readJsonFile(jsonFile()).getString("key")); + } + + /** + * "save" writes next to the file and moves the result over it. + *

+ * Verifies that the file holds what was stored before and what was added, and that no temporary + * file is left behind. + */ + @Test + public void save_replacesTheFileWhole_andLeavesNoTemporaryFile() { + setupJsonFile("{\"old\":\"value\"}"); + storage = new JsonFileStorage(jsonFile(), L); + storage.addAndSave("new", "value"); + + JSONObject saved = TestUtils.readJsonFile(jsonFile()); + Assert.assertEquals("value", saved.getString("old")); + Assert.assertEquals("value", saved.getString("new")); + Assert.assertFalse(JsonFileStorage.temporaryFileOf(jsonFile()).exists()); + } + + /** + * The temporary file of a save stopped before its move is deleted when the storage opens. + *

+ * Verifies that the storage reads the json file, which holds the previous save, and that the + * leftover file is gone, so it is not taken for other stored SDK data. + */ + @Test + public void constructor_deletesTheTemporaryFileOfAnInterruptedSave() throws IOException { + setupJsonFile("{\"saved\":\"before\"}"); + File leftover = JsonFileStorage.temporaryFileOf(jsonFile()); + Files.write(leftover.toPath(), "{\"saved\":".getBytes()); + + storage = new JsonFileStorage(jsonFile(), L); + + Assert.assertEquals("before", storage.getString("saved")); + Assert.assertFalse(leftover.exists()); + } + private static File jsonFile() { return new File(TestUtils.getTestSDirectory(), JSON_FILE_NAME); } diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/LogTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/LogTests.java index 9d7c95a4..965d7dcf 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/LogTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/LogTests.java @@ -1,5 +1,20 @@ package ly.count.sdk.java.internal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.List; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nonnull; +import ly.count.sdk.java.internal.ConfigurationProvider.LogGatheringState; +import org.json.JSONArray; +import org.json.JSONObject; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -12,6 +27,7 @@ import static ly.count.sdk.java.Config.LoggingLevel.ERROR; import static ly.count.sdk.java.Config.LoggingLevel.INFO; import static ly.count.sdk.java.Config.LoggingLevel.OFF; +import static ly.count.sdk.java.Config.LoggingLevel.VERBOSE; import static ly.count.sdk.java.Config.LoggingLevel.WARN; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -24,6 +40,43 @@ public class LogTests { private static final Throwable exception = new IllegalStateException("IAS"); private InternalConfig config; + private final List handedOver = new ArrayList<>(); + private final List queuedBatches = new CopyOnWriteArrayList<>(); + //the storage IDs of the batches the played request queue still holds, oldest first + private final Deque waitingBatchIds = new ConcurrentLinkedDeque<>(); + private final AtomicLong nextRequestId = new AtomicLong(1); + private final AtomicInteger queueAttempts = new AtomicInteger(); + private volatile boolean queueAccepts = true; + + /** + * Plays the request queue: records each batch, or refuses it while {@link #queueAccepts} is off, + * and holds it until {@link #sendOldestBatch(Log)}. + */ + private final Log.LogBatchQueue queue = new Log.LogBatchQueue() { + /** + * Records the batch and holds it, or refuses it. + */ + @Override + public Long queueLogBatch(@Nonnull String payload) { + queueAttempts.incrementAndGet(); + if (!queueAccepts) { + return null; + } + queuedBatches.add(new JSONObject(payload)); + long requestId = nextRequestId.getAndIncrement(); + waitingBatchIds.addLast(requestId); + return requestId; + } + + /** + * Whether the batch was not sent yet. + */ + @Override + public boolean isQueued(@Nonnull Long requestId) { + return waitingBatchIds.contains(requestId); + } + }; + @Before public void setupEveryTest() { String serverUrl = "http://www.serverurl.com"; @@ -33,6 +86,9 @@ public void setupEveryTest() { @After public void cleanupEveryTests() { + for (Log log : handedOver) { + log.endLogGathering(false); + } config = null; } @@ -40,4 +96,622 @@ public void cleanupEveryTests() { public void logInit_null() { Log log = new Log(null, null); } + + /** + * Every level method prints as the print level says and informs the listener, and it feeds the + * capture whatever the print level. + *

+ * Verifies, for a logger printing nothing, only errors and everything, that the listener hears + * every line with its level, that the print level stays as configured, that every line waits for + * its consent while no consent provider is set, and that an armed directive uploads them with + * their level characters. + */ + @Test + public void levelMethods_informTheListenerAndFeedTheCapture_whateverThePrintLevel() throws Exception { + for (Config.LoggingLevel printLevel : new Config.LoggingLevel[] { OFF, ERROR, VERBOSE }) { + List heard = new ArrayList<>(); + Log log = new Log(printLevel, (logMessage, level) -> heard.add(level + " " + logMessage)); + log.v("aa"); + log.d("bb"); + log.i("cc"); + log.w("dd"); + log.e("ee"); + + Assert.assertEquals(Arrays.asList("VERBOSE aa", "DEBUG bb", "INFO cc", "WARN dd", "ERROR ee"), heard); + Assert.assertEquals(printLevel, log.getPrintLevel()); + Assert.assertEquals(LogGatheringState.UNDECIDED, log.logGatheringState); + Assert.assertEquals(5, log.awaitingConsentLineCount()); + + queuedBatches.clear(); + handOver(log, settings(LogGatheringState.GATHERING, "gather_levels", ModuleConfiguration.logGatheringAllLevels, 100)); + log.onSdkInitFinished(); + awaitDelivery(log); + JSONArray lines = queuedBatches.get(0).getJSONArray("l"); + String[] expected = { "v aa", "d bb", "i cc", "w dd", "e ee" }; + for (int i = 0; i < expected.length; i++) { + JSONObject line = lines.getJSONObject(i); + Assert.assertEquals(expected[i], line.getString("l") + " " + line.getString("m")); + } + } + } + + /** + * The buffer holds at most 500 lines and 128 KB of characters, dropping the oldest lines and + * counting them, and a message is cut to what the server keeps. + *

+ * Verifies both ceilings with the drop count and the character count they leave, that the first + * batch reports every dropped line in {@code d}, and that a message is cut at 4096 characters, + * or one less where the cut would split a surrogate pair. + */ + @Test + public void buffer_holdsAtMost500LinesAnd128KB_droppingTheOldestIntoD_andCutsLongMessages() throws Exception { + Log lines = new Log(OFF, null); + handOver(lines, settings(LogGatheringState.UNDECIDED, null, ModuleConfiguration.logGatheringAllLevels, ModuleConfiguration.logGatheringDefaultBatchSize)); + int handOverLines = lines.heldLogLineCount(); + for (int i = 0; i < 600; i++) { + lines.e("line " + i); + } + Assert.assertEquals(ModuleConfiguration.logGatheringMaxBufferedLines, lines.heldLogLineCount()); + Assert.assertEquals(handOverLines + 100, lines.droppedLogLineCount); + + Log characters = new Log(OFF, null); + handOver(characters, settings(LogGatheringState.GATHERING, "gather_characters", "e", 500)); + String longest = repeat('x', Log.maxMessageLength); + for (int i = 0; i < 40; i++) { + characters.e(longest); + } + int fitting = Log.gatheredCharCeiling / Log.maxMessageLength; + Assert.assertEquals(fitting, characters.heldLogLineCount()); + Assert.assertEquals(40 - fitting, characters.droppedLogLineCount); + Assert.assertEquals(Log.gatheredCharCeiling, characters.gatheredChars); + + characters.onSdkInitFinished(); + awaitDelivery(characters); + Assert.assertEquals(1, queuedBatches.size()); + Assert.assertEquals(40 - fitting, queuedBatches.get(0).getInt("d")); + Assert.assertEquals(fitting, queuedBatches.get(0).getJSONArray("l").length()); + + Log cut = new Log(OFF, null); + cut.e(repeat('y', 5000)); + cut.e(repeat('z', Log.maxMessageLength - 1) + "\uD83D\uDE00 after the pair"); + cut.e("short"); + queuedBatches.clear(); + handOver(cut, settings(LogGatheringState.GATHERING, "gather_cut", "e", 10)); + cut.onSdkInitFinished(); + awaitDelivery(cut); + JSONArray cutLines = queuedBatches.get(0).getJSONArray("l"); + Assert.assertEquals(repeat('y', Log.maxMessageLength), cutLines.getJSONObject(0).getString("m")); + Assert.assertEquals(repeat('z', Log.maxMessageLength - 1), cutLines.getJSONObject(1).getString("m")); + Assert.assertEquals("short", cutLines.getJSONObject(2).getString("m")); + } + + /** + * Once something decides against gathering, capturing costs nothing: no line is held and no + * delivery thread is started. + */ + @Test + public void notGathering_holdsNothing_andStartsNoDeliveryThread() { + Log log = new Log(OFF, null); + log.e("held while undecided"); + Assert.assertEquals(1, log.awaitingConsentLineCount()); + + handOver(log, settings(LogGatheringState.NOT_GATHERING, null, ModuleConfiguration.logGatheringAllLevels, ModuleConfiguration.logGatheringDefaultBatchSize)); + Assert.assertEquals(LogGatheringState.NOT_GATHERING, log.logGatheringState); + Assert.assertFalse(log.isCapturingLogs()); + Assert.assertEquals(0, log.heldLogLineCount()); + Assert.assertEquals(0, log.awaitingConsentLineCount()); + + for (int i = 0; i < 1000; i++) { + log.e("error " + i); + log.v("verbose " + i); + } + log.flushGatheredLogs(); + Assert.assertEquals(0, log.heldLogLineCount()); + Assert.assertEquals(0, log.droppedLogLineCount); + Assert.assertEquals(0, log.gatheredChars); + Assert.assertNull(log.logDeliveryExecutor); + Assert.assertEquals(0, queueAttempts.get()); + } + + /** + * A line that carries an uploaded batch, and a line logged by a thread doing this feature's own + * transport work, are never held, while other threads are captured meanwhile. + */ + @Test + public void selfCapture_batchCarryingLinesAndLinesOfAMarkedThread_areNotHeld() throws InterruptedException { + Log log = new Log(OFF, null); + handOver(log, settings(LogGatheringState.UNDECIDED, null, ModuleConfiguration.logGatheringAllLevels, ModuleConfiguration.logGatheringDefaultBatchSize)); + int handOverLines = log.heldLogLineCount(); + log.d("New request 1: [1] app_key=key&" + Log.transportMarker + "%7B%22i%22%3A%22gather%22%7D"); + Assert.assertEquals(handOverLines, log.heldLogLineCount()); + + log.setOwnTransportWork(true); + log.i("logged while doing transport work"); + Assert.assertEquals(handOverLines, log.heldLogLineCount()); + Thread other = new Thread(() -> log.i("logged by another thread meanwhile")); + other.start(); + other.join(); + Assert.assertEquals(handOverLines + 1, log.heldLogLineCount()); + + log.setOwnTransportWork(false); + log.i("logged once the mark is cleared"); + Assert.assertEquals(handOverLines + 2, log.heldLogLineCount()); + } + + /** + * A batch the queue refuses goes back to the front of the buffer with its drop count, so the + * later deliveries, one batch at a time as each is sent, upload every line in order and still + * report the loss. + */ + @Test + public void refusedBatch_goesBackToTheFrontOfTheBuffer_withItsDropCount() throws Exception { + Log log = new Log(OFF, null); + handOver(log, settings(LogGatheringState.GATHERING, "gather_refused", "e", 100)); + for (int i = 0; i < 505; i++) { + log.e("line " + i); + } + queueAccepts = false; + log.onSdkInitFinished(); + awaitDelivery(log); + Assert.assertEquals(1, queueAttempts.get()); + Assert.assertEquals(ModuleConfiguration.logGatheringMaxBufferedLines, log.heldLogLineCount()); + Assert.assertEquals(5, log.droppedLogLineCount); + + queueAccepts = true; + log.flushGatheredLogs(); + awaitDelivery(log); + sendQueuedBatches(log); + Assert.assertEquals(0, log.heldLogLineCount()); + Assert.assertEquals(5, queuedBatches.size()); + List messages = new ArrayList<>(); + for (int b = 0; b < queuedBatches.size(); b++) { + Assert.assertEquals(b == 0 ? 5 : 0, queuedBatches.get(b).getInt("d")); + messages.addAll(messagesOf(queuedBatches.get(b))); + } + List expected = new ArrayList<>(); + for (int i = 5; i < 505; i++) { + expected.add("line " + i); + } + Assert.assertEquals(expected, messages); + } + + /** + * Ending the gathering queues what is held, synchronously, only when asked, and leaves a logger + * that neither captures nor queues anything, whatever is decided afterwards. + */ + @Test + public void endedGathering_queuesTheHeldLinesOnlyWhenAsked_andIgnoresEverythingAfterwards() throws Exception { + Log kept = new Log(OFF, null); + handOver(kept, settings(LogGatheringState.GATHERING, "gather_end", "e", 100)); + kept.onSdkInitFinished(); + kept.e("tail 1"); + kept.e("tail 2"); + kept.endLogGathering(true); + Assert.assertEquals(1, queuedBatches.size()); + Assert.assertEquals("gather_end", queuedBatches.get(0).getString("i")); + Assert.assertEquals(Arrays.asList("tail 1", "tail 2"), messagesOf(queuedBatches.get(0))); + Assert.assertEquals(LogGatheringState.NOT_GATHERING, kept.logGatheringState); + Assert.assertNull(kept.logDeliveryExecutor); + + kept.setLogGatheringProviders(settings(LogGatheringState.GATHERING, "gather_again", "e", 10), () -> ~0, queue); + kept.e("logged after the end"); + kept.flushGatheredLogs(); + Assert.assertEquals(LogGatheringState.NOT_GATHERING, kept.logGatheringState); + Assert.assertEquals(0, kept.heldLogLineCount()); + Assert.assertNull(kept.logDeliveryExecutor); + Assert.assertEquals(1, queueAttempts.get()); + + Log discarded = new Log(OFF, null); + handOver(discarded, settings(LogGatheringState.GATHERING, "gather_discarded", "e", 100)); + discarded.onSdkInitFinished(); + discarded.e("discarded line"); + discarded.endLogGathering(false); + Assert.assertEquals(1, queueAttempts.get()); + Assert.assertEquals(0, discarded.heldLogLineCount()); + Assert.assertEquals(LogGatheringState.NOT_GATHERING, discarded.logGatheringState); + } + + /** + * A line is gathered only once the consent of the feature it belongs to is given, found from + * the name it starts with in brackets, and a line of no feature needs both the events and the + * user profiles consent. Until then it waits apart, and it joins the gather in capture order. + *

+ * Verifies that every line waits before the providers are set; that with only the sessions + * consent given only the session lines are held; that each consent given releases exactly its + * lines, the line of no feature once both of its consents are given; that the batch keeps the + * order the lines were logged in; and that the waiting lines are capped at 500, oldest first, + * without counting into {@code d}. + */ + @Test + public void perLineConsent_gathersALineOnlyWithTheConsentOfItsFeature_inCaptureOrder() throws Exception { + Log log = new Log(OFF, null); + log.e("[ModuleLocation] setLocation, gps:[12.345678,98.765432]"); + log.e("[SessionImpl] begin"); + Assert.assertEquals(0, log.heldLogLineCount()); + Assert.assertEquals(2, log.awaitingConsentLineCount()); + + AtomicInteger given = new AtomicInteger(CoreFeature.Sessions.getIndex()); + handedOver.add(log); + log.setLogGatheringProviders(settings(LogGatheringState.GATHERING, "gather_consent", "e", 100), given::get, queue); + Assert.assertEquals(1, log.heldLogLineCount()); + log.e("[ModuleEvents] recordEventInternal, key:[purchase]"); + log.e("[Test] a line of no feature"); + log.e("[Views] startView, name:[home]"); + Assert.assertEquals(1, log.heldLogLineCount()); + Assert.assertEquals(4, log.awaitingConsentLineCount()); + + given.set(given.get() | CoreFeature.Events.getIndex()); + log.releaseConsentedLogLines(); + Assert.assertEquals(2, log.heldLogLineCount()); + given.set(given.get() | CoreFeature.UserProfiles.getIndex() | CoreFeature.Location.getIndex()); + log.releaseConsentedLogLines(); + Assert.assertEquals(4, log.heldLogLineCount()); + Assert.assertEquals(1, log.awaitingConsentLineCount()); + + log.onSdkInitFinished(); + awaitDelivery(log); + Assert.assertEquals(1, queuedBatches.size()); + Assert.assertEquals(Arrays.asList("[ModuleLocation] setLocation, gps:[12.345678,98.765432]", "[SessionImpl] begin", + "[ModuleEvents] recordEventInternal, key:[purchase]", "[Test] a line of no feature"), messagesOf(queuedBatches.get(0))); + + for (int i = 0; i < 520; i++) { + log.e("[ModuleViews] waiting line " + i); + } + Assert.assertEquals(ModuleConfiguration.logGatheringMaxBufferedLines, log.awaitingConsentLineCount()); + Assert.assertEquals(0, log.droppedLogLineCount); + } + + /** + * One batch waits in the request queue at a time: until it is sent the lines stay in the + * buffer, and each send lets the next batch go, a partial one included. + *

+ * Verifies that a delivery queues a single batch however many are held; that neither a full + * batch nor a flush queues another while it waits; that each sent batch lets exactly one more + * go, until every line went out once, in order; and that a refused batch does not count as + * waiting. + */ + @Test + public void oneBatchAtATime_holdsTheLinesUntilTheWaitingBatchIsSent() throws Exception { + Log log = new Log(OFF, null); + handOver(log, settings(LogGatheringState.GATHERING, "gather_one", "e", 10)); + for (int i = 0; i < 35; i++) { + log.e("line " + i); + } + log.onSdkInitFinished(); + awaitDelivery(log); + Assert.assertEquals(1, queuedBatches.size()); + Assert.assertEquals(1, waitingBatchIds.size()); + Assert.assertEquals(25, log.heldLogLineCount()); + + log.e("line 35"); + log.flushGatheredLogs(); + awaitDelivery(log); + Assert.assertEquals(1, queuedBatches.size()); + Assert.assertEquals(26, log.heldLogLineCount()); + + for (int sent = 1; sent <= 3; sent++) { + sendOldestBatch(log); + awaitDelivery(log); + Assert.assertEquals(sent + 1, queuedBatches.size()); + Assert.assertEquals(1, waitingBatchIds.size()); + } + sendOldestBatch(log); + awaitDelivery(log); + Assert.assertEquals(4, queuedBatches.size()); + Assert.assertEquals(0, waitingBatchIds.size()); + List messages = new ArrayList<>(); + for (JSONObject batch : queuedBatches) { + messages.addAll(messagesOf(batch)); + } + List expected = new ArrayList<>(); + for (int i = 0; i < 36; i++) { + expected.add("line " + i); + } + Assert.assertEquals(expected, messages); + + queueAccepts = false; + log.e("refused line"); + log.flushGatheredLogs(); + awaitDelivery(log); + Assert.assertEquals(0, waitingBatchIds.size()); + queueAccepts = true; + log.flushGatheredLogs(); + awaitDelivery(log); + Assert.assertEquals(5, queuedBatches.size()); + Assert.assertEquals(Arrays.asList("refused line"), messagesOf(queuedBatches.get(4))); + } + + /** + * The batch waiting in the request queue holds the next one back only while the queue holds it, + * and its send resumes only a delivery it held back. + *

+ * Verifies that a batch the queue dropped without sending, as when the queue is over its size, + * no longer holds the lines back on the next flush; that a send that held nothing back queues + * nothing, so a line logged meanwhile waits for the next flush; and that a send after a delivery + * it held back queues the line that waited, a partial batch. + */ + @Test + public void theWaitingBatch_holdsTheNextBackOnlyWhileQueued_andItsSendResumesOnlyADeliveryItHeldBack() throws Exception { + Log log = new Log(OFF, null); + handOver(log, settings(LogGatheringState.GATHERING, "gather_waiting", "e", 10)); + log.e("line 0"); + log.onSdkInitFinished(); + awaitDelivery(log); + log.e("line 1"); + log.flushGatheredLogs(); + awaitDelivery(log); + Assert.assertEquals(1, queuedBatches.size()); + Assert.assertEquals(1, log.heldLogLineCount()); + + waitingBatchIds.clear(); + log.flushGatheredLogs(); + awaitDelivery(log); + Assert.assertEquals(2, queuedBatches.size()); + Assert.assertEquals(Collections.singletonList("line 1"), messagesOf(queuedBatches.get(1))); + + log.e("line 2"); + sendOldestBatch(log); + awaitDelivery(log); + Assert.assertEquals(2, queuedBatches.size()); + Assert.assertEquals(1, log.heldLogLineCount()); + + log.flushGatheredLogs(); + awaitDelivery(log); + log.e("line 3"); + log.flushGatheredLogs(); + awaitDelivery(log); + Assert.assertEquals(3, queuedBatches.size()); + sendOldestBatch(log); + awaitDelivery(log); + Assert.assertEquals(4, queuedBatches.size()); + Assert.assertEquals(Collections.singletonList("line 3"), messagesOf(queuedBatches.get(3))); + } + + /** + * A removed consent drops the held lines that needed it, so nothing gathered for a feature goes + * out once its consent is withdrawn, and the lines dropped are reported in {@code d}. + */ + @Test + public void removedConsent_dropsTheHeldLinesThatNeededIt_countingThemIntoD() throws Exception { + Log log = new Log(OFF, null); + AtomicInteger given = new AtomicInteger(CoreFeature.Events.getIndex() | CoreFeature.Location.getIndex() | CoreFeature.UserProfiles.getIndex()); + handedOver.add(log); + log.setLogGatheringProviders(settings(LogGatheringState.GATHERING, "gather_withdrawn", "e", 100), given::get, queue); + log.e("[ModuleEvents] an events line"); + log.e("[ModuleLocation] a location line"); + log.e("[Test] a line of no feature"); + Assert.assertEquals(3, log.heldLogLineCount()); + + given.set(CoreFeature.Events.getIndex()); + log.dropLinesWithoutConsent(); + + Assert.assertEquals(1, log.heldLogLineCount()); + log.onSdkInitFinished(); + awaitDelivery(log); + Assert.assertEquals(Collections.singletonList("[ModuleEvents] an events line"), messagesOf(queuedBatches.get(0))); + Assert.assertEquals(2, queuedBatches.get(0).getInt("d")); + } + + /** + * A gather the server stops queues the lines it still holds once, with its id, while a gather + * replaced by another drops them, and so does a decision against gathering made while nothing + * was decided. + *

+ * Verifies that stopping a gather queues one batch of exactly its held lines, with its id and + * past a batch already waiting; that the capture stops; that a gather replaced by a new id + * queues nothing; and that deciding off while undecided queues nothing. + */ + @Test + public void stoppedGather_queuesItsHeldLinesOnceWithItsId_whileAReplacedGatherDropsThem() throws Exception { + ModuleConfiguration stopped = settings(LogGatheringState.GATHERING, "gather_tail", "e", 10); + Log log = new Log(OFF, null); + handOver(log, stopped); + log.onSdkInitFinished(); + for (int i = 0; i < 13; i++) { + log.e("line " + i); + } + awaitDelivery(log); + Assert.assertEquals(1, queuedBatches.size()); + Assert.assertEquals(3, log.heldLogLineCount()); + + stopped.currentVLogGathering = ModuleConfiguration.LogGatheringDirective.NOT_GATHERING; + log.applyLogGatheringDirective("test"); + Assert.assertTrue("the held lines were not queued", TestUtils.waitFor(10_000, () -> queuedBatches.size() == 2)); + Assert.assertEquals("gather_tail", queuedBatches.get(1).getString("i")); + Assert.assertEquals(0, queuedBatches.get(1).getInt("d")); + Assert.assertEquals(Arrays.asList("line 10", "line 11", "line 12"), messagesOf(queuedBatches.get(1))); + Assert.assertFalse(log.isCapturingLogs()); + log.e("logged after the stop"); + Assert.assertEquals(0, log.heldLogLineCount()); + + ModuleConfiguration replaced = settings(LogGatheringState.GATHERING, "gather_one", "e", 100); + Log other = new Log(OFF, null); + handOver(other, replaced); + other.onSdkInitFinished(); + awaitDelivery(other); + other.e("line of the first gather"); + replaced.currentVLogGathering = new ModuleConfiguration.LogGatheringDirective(LogGatheringState.GATHERING, "gather_two", "e", 100); + other.applyLogGatheringDirective("test"); + awaitDelivery(other); + Assert.assertEquals(0, other.heldLogLineCount()); + + Log undecided = new Log(OFF, null); + undecided.e("line held while undecided"); + handOver(undecided, settings(LogGatheringState.NOT_GATHERING, null, ModuleConfiguration.logGatheringAllLevels, ModuleConfiguration.logGatheringDefaultBatchSize)); + undecided.onSdkInitFinished(); + Thread.sleep(100); + Assert.assertEquals(2, queuedBatches.size()); + Assert.assertEquals(2, queueAttempts.get()); + } + + /** + * Before the device ID changes without merge the gather stops: every held line is queued on + * the calling thread, past a batch already waiting, the lines waiting for their consent are + * dropped, and a later directive can still arm a new gather. + *

+ * Verifies that the held lines are queued before the call returns, in batches of at most + * {@code b} and in order; that a line waiting for its consent is not queued; that nothing is + * captured afterwards; and that a directive applied later arms a gather again. + */ + @Test + public void deviceIdChange_queuesTheHeldLinesThenStops_andALaterDirectiveCanArmAgain() throws Exception { + AtomicInteger given = new AtomicInteger(Log.UNATTRIBUTED_LINE_CONSENT); + ModuleConfiguration settings = settings(LogGatheringState.GATHERING, "gather_did", "e", 10); + Log log = new Log(OFF, null); + handedOver.add(log); + log.setLogGatheringProviders(settings, given::get, queue); + log.onSdkInitFinished(); + for (int i = 0; i < 25; i++) { + log.e("[Test] user A line " + i); + } + log.e("[ModuleLocation] user A location"); + awaitDelivery(log); + Assert.assertEquals(1, queuedBatches.size()); + Assert.assertEquals(1, log.awaitingConsentLineCount()); + + log.stopLogGatheringForDeviceIdChange(); + Assert.assertEquals(3, queuedBatches.size()); + List messages = new ArrayList<>(); + for (JSONObject batch : queuedBatches) { + Assert.assertEquals("gather_did", batch.getString("i")); + Assert.assertTrue(batch.getJSONArray("l").length() <= 10); + messages.addAll(messagesOf(batch)); + } + List expected = new ArrayList<>(); + for (int i = 0; i < 25; i++) { + expected.add("[Test] user A line " + i); + } + Assert.assertEquals(expected, messages); + Assert.assertEquals(LogGatheringState.NOT_GATHERING, log.logGatheringState); + Assert.assertEquals(0, log.awaitingConsentLineCount()); + log.e("[Test] user B line"); + Assert.assertEquals(0, log.heldLogLineCount()); + Assert.assertNull(log.logDeliveryExecutor); + + settings.currentVLogGathering = new ModuleConfiguration.LogGatheringDirective(LogGatheringState.GATHERING, "gather_b", "e", 10); + log.applyLogGatheringDirective("test"); + Assert.assertTrue(log.isGatheringLogs()); + log.e("[Test] user B armed line"); + Assert.assertEquals(1, log.heldLogLineCount()); + } + + /** + * A batch is valid JSON with its keys always in the same order, every message escaped. + */ + @Test + public void batchPayload_isJsonWithItsKeysInOrder_andEscapesEveryMessage() { + String awkward = "quote \" backslash \\ newline \n tab \t closing pair \uD83D\uDE00"; + List lines = Arrays.asList(new Log.LogLine(0, 1_755_500_000_000L, 'e', awkward, 0), new Log.LogLine(1, 1_755_500_000_001L, 'd', "", 0)); + String payload = Log.buildLogBatchPayload("gather \"quoted\"", 3, lines); + Assert.assertTrue(payload, payload.startsWith("{\"i\":\"gather \\\"quoted\\\"\",\"d\":3,\"l\":[{\"t\":1755500000000,\"l\":\"e\",\"m\":\"")); + + JSONObject batch = new JSONObject(payload); + Assert.assertEquals("gather \"quoted\"", batch.getString("i")); + Assert.assertEquals(3, batch.getInt("d")); + JSONArray parsed = batch.getJSONArray("l"); + Assert.assertEquals(2, parsed.length()); + Assert.assertEquals(1_755_500_000_000L, parsed.getJSONObject(0).getLong("t")); + Assert.assertEquals("e", parsed.getJSONObject(0).getString("l")); + Assert.assertEquals(awkward, parsed.getJSONObject(0).getString("m")); + Assert.assertEquals("d", parsed.getJSONObject(1).getString("l")); + Assert.assertEquals("", parsed.getJSONObject(1).getString("m")); + + Assert.assertEquals("{\"i\":\"gather\",\"d\":0,\"l\":[]}", Log.buildLogBatchPayload("gather", 0, Collections.emptyList())); + } + + /** + * Settings that hold nothing but the given log gathering directive. + * + * @param state the decision + * @param gatherId the gather id + * @param levels the level characters + * @param batchSize the batch size + * @return the settings + */ + private static ModuleConfiguration settings(LogGatheringState state, String gatherId, String levels, int batchSize) { + ModuleConfiguration settings = new ModuleConfiguration(); + settings.currentVLogGathering = new ModuleConfiguration.LogGatheringDirective(state, gatherId, levels, batchSize); + return settings; + } + + /** + * Hands a logger the given settings, every consent given and {@link #queue}, as an SDK + * that requires no consent does once its settings are resolved. Nothing is uploaded before the + * test tells the logger that init finished. + * + * @param log the logger + * @param settings the settings + */ + private void handOver(Log log, ModuleConfiguration settings) { + handedOver.add(log); + log.setLogGatheringProviders(settings, () -> ~0, queue); + } + + /** + * Waits until every delivery the logger was asked for so far ran. + * + * @param log the logger + * @throws Exception when the delivery thread does not finish in time + */ + private static void awaitDelivery(Log log) throws Exception { + ExecutorService executor = log.logDeliveryExecutor; + if (executor != null) { + executor.submit(() -> { + }).get(10, TimeUnit.SECONDS); + } + } + + /** + * Plays the request queue sending every batch the logger queued, one at a time, each send + * letting the logger queue its next batch, until none waits. + * + * @param log the logger + * @throws Exception when a delivery does not finish in time + */ + private void sendQueuedBatches(Log log) throws Exception { + for (int i = 0; i < 1000 && !waitingBatchIds.isEmpty(); i++) { + sendOldestBatch(log); + awaitDelivery(log); + } + } + + /** + * Plays the request queue sending its oldest batch, then telling the logger. + * + * @param log the logger + */ + private void sendOldestBatch(Log log) { + waitingBatchIds.pollFirst(); + log.onLogBatchSent(); + } + + /** + * The messages of a batch, in order. + * + * @param batch the batch + * @return the messages + */ + private static List messagesOf(JSONObject batch) { + List messages = new ArrayList<>(); + JSONArray lines = batch.getJSONArray("l"); + for (int i = 0; i < lines.length(); i++) { + messages.add(lines.getJSONObject(i).getString("m")); + } + return messages; + } + + /** + * A text of one character repeated. + * + * @param character the character + * @param count how many times + * @return the text + */ + private static String repeat(char character, int count) { + StringBuilder text = new StringBuilder(count); + for (int i = 0; i < count; i++) { + text.append(character); + } + return text.toString(); + } } diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationContentTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationContentTests.java new file mode 100644 index 00000000..b04fd278 --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationContentTests.java @@ -0,0 +1,842 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * The content settings and the journey triggers of the SDK behavior settings: {@code ecz}, + * {@code czi}, {@code rcz}, {@code jte} and {@code jtv}, observed through the public API. + *

+ * Settings requests and content fetches go through a {@link SettingsAndContentServer}, which answers + * on the calling thread. The zone timer's first tick is held an hour away, so the zone fetches only + * when a test ticks it by hand, and a tick that fetches proves the zone is active. The journey + * scenarios send the event requests through the real networking stack to a local HTTP server, since + * the content zone is refreshed only after the server answers the request carrying the trigger. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationContentTests { + + private static final String CONTENT_ENDPOINT = "/o/sdk/content?"; + private static final String NO_CONTENT_RESPONSE = "{\"jsonArray\":[{\"result\":\"No content block found!\"}]}"; + private static final String CONTENT_RESPONSE = "{\"html\":\"https://content.count.ly/block-1\",\"geo\":{" + + "\"p\":{\"x\":10,\"y\":20,\"w\":300,\"h\":400},\"l\":{\"x\":30,\"y\":40,\"w\":500,\"h\":600}}}"; + private static final long WAIT_BUDGET_MS = 30_000; + + private SettingsAndContentServer server; + private QueueServer queueServer; + private FakeDisplay display; + private final List logs = new CopyOnWriteArrayList<>(); + + /** + * Starts every test from empty storage, without a test module and with fresh fakes. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new SettingsAndContentServer(); + queueServer = null; + display = new FakeDisplay(); + logs.clear(); + } + + /** + * Stops the SDK, clearing its data, and the local HTTP server when one was started. + */ + @After + public void afterTest() { + Countly.instance().halt(); + if (queueServer != null) { + queueServer.http.stop(0); + } + } + + // region scenarios + + /** + * {@code ecz} enters the content zone once a display is registered, and turning it off leaves + * only a zone it entered. + *

+ * Verifies that the response of the fetch made at init, applied before the content module + * finished its init, enters nothing while no display is registered; that registering one enters + * the zone, which a refresh keeps as the settings' zone; that turning the setting off leaves that + * zone and turning it on enters again; that a zone the application entered, or claimed by entering + * while the settings' zone ran, survives the setting turning off; that a response which leaves the + * setting as it is changes nothing, so a zone the application left stays left; that registering a + * display again enters; and that the stored setting enters again after a restart. + */ + @Test + public void contentZoneEnabled_entersOnceADisplayIsRegistered_andTurningItOffLeavesOnlyTheZoneItEntered() { + server.settings.respondWith(new ServerConfigBuilder().contentZone(true)); + init(contentConfig()); + holdTheZoneTimer(); + + Assert.assertTrue(provider().getContentZoneEnabled()); + Assert.assertEquals(2, logsContaining("it is entered once a content display is registered")); + assertZoneIsInactive(0); + + Countly.instance().content().setContentDisplay(display); + assertZoneFetches(1); + Countly.instance().content().refreshContentZone(); + assertZoneFetches(2); + + push(new ServerConfigBuilder().contentZone(false)); + assertZoneIsInactive(2); + push(new ServerConfigBuilder().contentZone(true)); + assertZoneFetches(3); + + Countly.instance().content().enterContentZone(); + push(new ServerConfigBuilder().contentZone(false)); + assertZoneFetches(4); + push(new ServerConfigBuilder().eventQueueSize(5)); + assertZoneFetches(5); + Countly.instance().content().exitContentZone(); + assertZoneIsInactive(5); + + Countly.instance().content().enterContentZone(); + push(new ServerConfigBuilder().contentZone(true)); + push(new ServerConfigBuilder().contentZone(false)); + assertZoneFetches(6); + Countly.instance().content().exitContentZone(); + + push(new ServerConfigBuilder().contentZone(true)); + assertZoneFetches(7); + Countly.instance().content().exitContentZone(); + push(new ServerConfigBuilder().contentZone(true).eventQueueSize(6)); + assertZoneIsInactive(7); + + Countly.instance().content().setContentDisplay(display); + assertZoneFetches(8); + Countly.instance().stop(); + + server.settings.pending(); + init(contentConfig()); + holdTheZoneTimer(); + assertZoneIsInactive(8); + Countly.instance().content().setContentDisplay(display); + assertZoneFetches(9); + } + + /** + * {@code czi} is the fetch interval of a zone, read when the zone is entered. + *

+ * Verifies that without the setting the developer's interval is used, that the setting overrides + * it, that a response changing it restarts the timer of the active zone at the new interval while + * the zone keeps fetching, that a response repeating it keeps the running timer, and that a change + * while no zone is active starts nothing and applies when a zone is entered. + */ + @Test + public void contentZoneInterval_isReadWhenTheZoneIsEntered_andAChangeRestartsTheTimerOfTheActiveZone() { + server.settings.pending(); + init(contentConfig().content.setZoneTimerInterval(20)); + holdTheZoneTimer(); + Countly.instance().content().setContentDisplay(display); + Countly.instance().content().enterContentZone(); + Assert.assertEquals(20, zoneTimer().getTimerDelaySeconds()); + Countly.instance().halt(); + + server.settings.respondWith(new ServerConfigBuilder().contentZoneInterval(60)); + init(contentConfig().content.setZoneTimerInterval(20)); + holdTheZoneTimer(); + Assert.assertEquals(60, provider().getContentZoneTimerInterval()); + Countly.instance().content().setContentDisplay(display); + Countly.instance().content().enterContentZone(); + CountlyTimer atEnter = zoneTimer(); + Assert.assertEquals(60, atEnter.getTimerDelaySeconds()); + assertZoneFetches(1); + + push(new ServerConfigBuilder().contentZoneInterval(90)); + CountlyTimer restarted = zoneTimer(); + Assert.assertNotSame(atEnter, restarted); + Assert.assertEquals(90, restarted.getTimerDelaySeconds()); + assertZoneFetches(2); + + push(new ServerConfigBuilder().contentZoneInterval(90).eventQueueSize(3)); + Assert.assertSame(restarted, zoneTimer()); + + Countly.instance().content().exitContentZone(); + push(new ServerConfigBuilder().contentZoneInterval(120)); + Assert.assertNull(zoneTimer()); + Countly.instance().content().enterContentZone(); + Assert.assertEquals(120, zoneTimer().getTimerDelaySeconds()); + assertZoneFetches(3); + } + + /** + * {@code jte} and {@code jtv} send the whole event queue as soon as a trigger is recorded. + *

+ * Verifies on disk, with networking off, that events that are not triggers wait for the queue + * threshold, an internal event listed in {@code jte} included, that a custom trigger sends them all + * with it, that each later trigger gets its own request and each such request waits for its + * response, that both the start and the end of a trigger view send the queue, and that a trigger + * filtered out by {@code eb} or a trigger view while view tracking is off sends nothing. + */ + @Test + public void journeyTriggers_sendTheWholeEventQueueRightAway_eachTriggerInItsOwnRequest() { + server.settings.respondWith(journeyTriggerSettings()); + init(contentConfig().enableFeatures(Config.Feature.Views)); + + Countly.instance().events().recordEvent("regular"); + Countly.instance().events().recordEvent("[CLY]_star_rating"); + Countly.instance().views().startView("home"); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + TestUtils.validateEQSize(3); + + Countly.instance().events().recordEvent("purchase", TestUtils.map("item", "book")); + TestUtils.letTheClockCatchUp(); + TestUtils.validateEQSize(0); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + List sent = TestUtils.readEventsFromRequest(0, TestUtils.DEVICE_ID); + Assert.assertEquals(4, sent.size()); + Assert.assertEquals("regular", sent.get(0).key); + Assert.assertEquals("[CLY]_star_rating", sent.get(1).key); + Assert.assertEquals(ModuleViews.KEY_VIEW_EVENT, sent.get(2).key); + Assert.assertEquals("purchase", sent.get(3).key); + Assert.assertEquals("book", sent.get(3).segmentation.get("item")); + + Countly.instance().events().recordEvent("purchase"); + TestUtils.letTheClockCatchUp(); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + Assert.assertEquals(1, TestUtils.readEventsFromRequest(1, TestUtils.DEVICE_ID).size()); + + Countly.instance().views().startView("checkout"); + TestUtils.letTheClockCatchUp(); + Countly.instance().views().stopViewWithName("checkout"); + TestUtils.letTheClockCatchUp(); + Assert.assertEquals(4, TestUtils.getCurrentRQ().length); + List checkoutStart = TestUtils.readEventsFromRequest(2, TestUtils.DEVICE_ID); + List checkoutEnd = TestUtils.readEventsFromRequest(3, TestUtils.DEVICE_ID); + Assert.assertEquals("checkout", checkoutStart.get(0).segmentation.get(ModuleViews.KEY_NAME)); + Assert.assertEquals("1", checkoutStart.get(0).segmentation.get(ModuleViews.KEY_VISIT)); + Assert.assertEquals("checkout", checkoutEnd.get(0).segmentation.get(ModuleViews.KEY_NAME)); + Assert.assertNull(checkoutEnd.get(0).segmentation.get(ModuleViews.KEY_VISIT)); + Assert.assertEquals(storedRequestIds(), SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds); + + Countly.instance().views().startView("profile"); + push(journeyTriggerSettings().eventFilterList(names("purchase"), false)); + Countly.instance().events().recordEvent("purchase"); + push(journeyTriggerSettings().viewTracking(false)); + Assert.assertNull(Countly.instance().views().startView("checkout")); + Assert.assertEquals(4, TestUtils.getCurrentRQ().length); + TestUtils.validateEQSize(1); + Assert.assertEquals(4, SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds.size()); + } + + /** + * A {@code jte} trigger refreshes the content zone once the server accepts the request carrying it. + *

+ * Verifies against a local server that the zone is entered only after the request carrying the + * trigger and the events queued before it was accepted; that while a content block is on screen + * the refresh is ignored; that a rejected request refreshes nothing, and neither does its retry + * once accepted; and that an attempt that got no response settles it the same way. + */ + @Test + public void journeyTriggerEvent_refreshesTheZoneOnlyWhenTheServerAcceptsItsRequest() throws Exception { + startQueueServer(); + server.settings.respondWith(new ServerConfigBuilder().journeyTriggerEvents(names("purchase"))); + init(networkedContentConfig()); + holdTheZoneTimer(); + Countly.instance().content().setContentDisplay(display); + assertZoneIsInactive(0); + + Countly.instance().events().recordEvent("regular"); + TestUtils.validateEQSize(1); + Countly.instance().events().recordEvent("purchase"); + awaitZoneFetch(1); + Assert.assertEquals(1, queueServer.eventRequests().size()); + Assert.assertEquals("[\"regular\",\"purchase\"]", eventKeys(queueServer.eventRequests().get(0)).toString()); + awaitEmptyQueue(); + + server.contentResponse = CONTENT_RESPONSE; + assertZoneFetches(2); + Assert.assertEquals(1, display.presented.size()); + Countly.instance().events().recordEvent("purchase"); + awaitLogs("a content block is on screen, ignoring the call", 1); + display.lastCallback.onClosed(new HashMap<>()); + Countly.instance().content().exitContentZone(); + server.contentResponse = NO_CONTENT_RESPONSE; + awaitEmptyQueue(); + assertZoneIsInactive(2); + + queueServer.mode = QueueServer.REJECT; + Countly.instance().events().recordEvent("purchase"); + awaitLogs("carrying a journey trigger failed with code [500]", 1); + Assert.assertEquals(1, queuedRequestCount()); + assertZoneIsInactive(2); + queueServer.mode = QueueServer.ACCEPT; + awaitEmptyQueue(); + assertZoneIsInactive(2); + + queueServer.mode = QueueServer.DROP; + Countly.instance().events().recordEvent("purchase"); + awaitLogs("carrying a journey trigger failed with code [" + Transport.NO_RESPONSE_CODE + "]", 1); + assertZoneIsInactive(2); + queueServer.mode = QueueServer.ACCEPT; + awaitEmptyQueue(); + assertZoneIsInactive(2); + Assert.assertEquals(2, logsContaining("carrying a journey trigger, refreshing the content zone")); + Assert.assertTrue(SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds.isEmpty()); + } + + /** + * A {@code jtv} trigger refreshes the zone like an event does, it never triggers while view + * tracking is off, and {@code rcz} off ignores every refresh. + *

+ * Verifies against a local server that with refreshing forbidden the application's refresh is + * ignored before it sends the queued events and the refresh of an accepted trigger is ignored too; + * that with view tracking off a trigger view is not recorded, so nothing is sent; and that with + * both allowed a trigger view sends the queue and enters the zone once the server accepted it. + */ + @Test + public void journeyTriggerView_refreshesLikeAnEvent_neverWhileViewTrackingIsOff_andRczOffIgnoresEveryRefresh() throws Exception { + startQueueServer(); + server.settings.respondWith(new ServerConfigBuilder() + .journeyTriggerViews(names("checkout")) + .journeyTriggerEvents(names("purchase")) + .refreshContentZone(false)); + init(networkedContentConfig()); + holdTheZoneTimer(); + Countly.instance().content().setContentDisplay(display); + + Countly.instance().views().startView("home"); + Countly.instance().content().refreshContentZone(); + TestUtils.validateEQSize(1); + Assert.assertEquals(1, logsContaining("refreshing the content zone is disabled by the SDK behavior settings")); + assertZoneIsInactive(0); + + Countly.instance().events().recordEvent("purchase"); + awaitLogs("refreshing the content zone is disabled by the SDK behavior settings", 2); + awaitEmptyQueue(); + assertZoneIsInactive(0); + Assert.assertEquals("[\"[CLY]_view\",\"purchase\"]", eventKeys(queueServer.eventRequests().get(0)).toString()); + + push(new ServerConfigBuilder().refreshContentZone(true).viewTracking(false) + .journeyTriggerViews(names("checkout")) + .journeyTriggerEvents(names("purchase"))); + Assert.assertNull(Countly.instance().views().startView("checkout")); + TestUtils.validateEQSize(0); + Assert.assertEquals(0, queuedRequestCount()); + assertZoneIsInactive(0); + + push(new ServerConfigBuilder().viewTracking(true) + .journeyTriggerViews(names("checkout")) + .journeyTriggerEvents(names("purchase"))); + Countly.instance().views().startView("checkout"); + awaitZoneFetch(1); + List> eventRequests = queueServer.eventRequests(); + Assert.assertEquals(2, eventRequests.size()); + JSONObject view = new JSONArray(eventRequests.get(1).get("events")).getJSONObject(0); + Assert.assertEquals(ModuleViews.KEY_VIEW_EVENT, view.getString("key")); + Assert.assertEquals("checkout", view.getJSONObject("segmentation").getString(ModuleViews.KEY_NAME)); + awaitEmptyQueue(); + } + + /** + * A settings change turning {@code ecz} on that arrives while the SDK stops starts no zone timer. + *

+ * Verifies, with the change handed over right after the content module left its zone during + * {@code halt()}, while its display is still registered, that the stopping SDK dispatches the + * change to no module, and that the content module, told directly, enters no zone, so no zone + * timer outlives the SDK. + */ + @Test + public void contentZoneEnabled_arrivingWhileTheSdkStops_startsNoZoneTimer() { + AtomicInteger dispatchedChanges = new AtomicInteger(); + SDKCore.testDummyModule = new ModuleBase() { + /** + * Counts a dispatched settings change. + * + * @param config configuration of the running SDK + */ + @Override + protected void onSdkConfigurationChanged(InternalConfig config) { + dispatchedChanges.incrementAndGet(); + } + }; + AtomicReference onZoneLeft = new AtomicReference<>(); + server.settings.respondWith(new ServerConfigBuilder()); + init(TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Content, Config.Feature.Events) + .setLogListener((message, level) -> { + Runnable hook = message != null && message.startsWith("[ModuleContent] exitContentZoneInternal, left the content zone") ? onZoneLeft.getAndSet(null) : null; + if (hook != null) { + hook.run(); + } + })); + Countly.instance().content().setContentDisplay(display); + ModuleContent content = contentModule(); + ModuleConfiguration configuration = SDKCore.instance.module(ModuleConfiguration.class); + InternalConfig config = SDKCore.instance.config; + Assert.assertNull(content.contentTimer); + int dispatchedBeforeStop = dispatchedChanges.get(); + + onZoneLeft.set(() -> { + configuration.currentVContentZone = true; + config.sdk.onSdkConfigurationChanged(config); + content.onSdkConfigurationChanged(config); + }); + try { + Countly.instance().halt(); + Assert.assertNull("the change was never handed over", onZoneLeft.get()); + Assert.assertEquals(dispatchedBeforeStop, dispatchedChanges.get()); + Assert.assertNull(content.contentTimer); + } finally { + SDKCore.testDummyModule = null; + CountlyTimer leaked = content.contentTimer; + if (leaked != null) { + leaked.stopTimer(false); + } + } + } + + // endregion + // region helpers + + /** + * The settings of the journey trigger scenario: two trigger events, one trigger view, a queue + * that waits for 100 events and networking off, so every request stays on disk. Each response + * replaces the stored settings, so every response of the scenario starts from these. + * + * @return a builder holding those settings + */ + private static ServerConfigBuilder journeyTriggerSettings() { + return new ServerConfigBuilder() + .journeyTriggerEvents(names("purchase", "[CLY]_star_rating")) + .journeyTriggerViews(names("checkout")) + .eventQueueSize(100) + .networking(false); + } + + /** + * Stands in for the settings and the content endpoints of the server. Settings requests go to a + * {@link ModuleConfigurationTests.ServerConfigResponder}; content fetches are recorded and answered + * on the calling thread with {@link #contentResponse}, unless networking is off, where the SDK's own + * request maker sends nothing. + */ + private static final class SettingsAndContentServer implements ImmediateRequestGenerator { + final ModuleConfigurationTests.ServerConfigResponder settings = new ModuleConfigurationTests.ServerConfigResponder(); + final List contentFetches = new CopyOnWriteArrayList<>(); + volatile String contentResponse = NO_CONTENT_RESPONSE; + + /** + * A request maker that routes by endpoint. + * + * @return the request maker + */ + @Override + public ImmediateRequestI createImmediateRequestMaker() { + ImmediateRequestI settingsRequestMaker = settings.createImmediateRequestMaker(); + return (requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log) -> { + if (!CONTENT_ENDPOINT.equals(customEndpoint)) { + settingsRequestMaker.doWork(requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log); + return; + } + if (!networkingIsEnabled) { + callback.callback(null); + return; + } + contentFetches.add(requestData); + callback.callback(new JSONObject(contentResponse)); + }; + } + } + + /** + * A local HTTP server standing in for the request queue endpoint {@code /i}: it records the + * parameters of every request and accepts it, rejects it with a 500, or drops the connection + * without any response. + */ + private static final class QueueServer { + static final int ACCEPT = 0; + static final int REJECT = 1; + static final int DROP = 2; + + final HttpServer http; + final List> received = new CopyOnWriteArrayList<>(); + volatile int mode = ACCEPT; + + /** + * Starts the server on a free loopback port. + * + * @throws IOException when the server cannot start + */ + QueueServer() throws IOException { + http = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + http.createContext("/", this::handle); + http.setExecutor(Executors.newSingleThreadExecutor()); + http.start(); + } + + /** + * Records a request and answers it as {@link #mode} says. + * + * @param exchange the request and its response + * @throws IOException when the response cannot be written + */ + private void handle(HttpExchange exchange) throws IOException { + String query = exchange.getRequestURI().getRawQuery(); + String body = new String(readAll(exchange.getRequestBody()), StandardCharsets.UTF_8); + received.add(parseParams(query != null && !query.isEmpty() ? query : body)); + + int currentMode = mode; + if (currentMode == DROP) { + exchange.close(); + return; + } + + byte[] response = (currentMode == ACCEPT ? "{\"result\":\"Success\"}" : "{\"result\":\"Server error\"}").getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(currentMode == ACCEPT ? 200 : 500, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + + /** + * The received requests that carry events, in the order they arrived. + * + * @return the parameters of each request + */ + List> eventRequests() { + List> requests = new ArrayList<>(); + for (Map params : received) { + if (params.containsKey("events")) { + requests.add(params); + } + } + return requests; + } + + /** + * The port the server listens on. + * + * @return the port + */ + int port() { + return http.getAddress().getPort(); + } + } + + /** + * A display that records what it was asked to show and lets a test close it. + */ + private static final class FakeDisplay implements ContentDisplay { + final List presented = new CopyOnWriteArrayList<>(); + volatile ContentCloseCallback lastCallback; + + /** + * A landscape screen. + * + * @return the screen + */ + @Override + public ContentScreen getScreen() { + return new ContentScreen(1600, 900); + } + + /** + * Records the content and keeps the callback that closes it. + * + * @param content the content to show + * @param onClosed called when the content closes + */ + @Override + public void present(ContentData content, ContentCloseCallback onClosed) { + presented.add(content); + lastCallback = onClosed; + } + } + + /** + * Initializes the SDK with {@link #server} answering its settings requests and content fetches. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + server.settings.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * The settings test configuration with content and events, keeping every log line. + * + * @return the configuration + */ + private Config contentConfig() { + return TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Content, Config.Feature.Events) + .setLogListener((message, level) -> logs.add(message)); + } + + /** + * A configuration pointing at {@link #queueServer}, with content, events and views, an event + * queue that only a trigger flushes, no request cooldowns, and every log line kept. + * + * @return the configuration + */ + private Config networkedContentConfig() { + File directory = TestUtils.getTestSDirectory(); + TestUtils.checkSdkStorageRootDirectoryExist(directory); + return new Config("http://127.0.0.1:" + queueServer.port(), TestUtils.SERVER_APP_KEY, directory) + .setApplicationVersion(TestUtils.APPLICATION_VERSION) + .setCustomDeviceId(TestUtils.DEVICE_ID) + .enableFeatures(Config.Feature.Content, Config.Feature.Events, Config.Feature.Views) + .setEventQueueSizeToSend(100) + .setNetworkRequestCooldown(0) + .setNetworkImportantRequestCooldown(0) + .setLogListener((message, level) -> logs.add(message)); + } + + /** + * Starts {@link #queueServer}, accepting every request. + * + * @throws IOException when the server cannot start + */ + private void startQueueServer() throws IOException { + queueServer = new QueueServer(); + } + + /** + * Holds the first tick of every zone timer an hour away, so the zone fetches only when a test + * ticks it by hand. + */ + private static void holdTheZoneTimer() { + contentModule().startedAtForTests(System.currentTimeMillis() + 60L * 60L * 1000L); + } + + /** + * The content module of the running SDK. + * + * @return the module + */ + private static ModuleContent contentModule() { + return SDKCore.instance.module(ModuleContent.class); + } + + /** + * The timer of the active zone. + * + * @return the timer, {@code null} while no zone is active + */ + private static CountlyTimer zoneTimer() { + return contentModule().contentTimer; + } + + /** + * Ticks the zone once by hand and asserts the total number of content fetches after the tick. + * + * @param expectedFetches the fetches expected so far + */ + private void assertZoneFetches(int expectedFetches) { + contentModule().onZoneTimerTick(); + Assert.assertEquals(expectedFetches, server.contentFetches.size()); + } + + /** + * Ticks the zone once by hand and asserts that it fetched nothing, as no zone is active. + * + * @param fetchesSoFar the fetches made so far + */ + private void assertZoneIsInactive(int fetchesSoFar) { + assertZoneFetches(fetchesSoFar); + } + + /** + * Ticks the zone by hand until it fetches, which it does once a refresh entered it. + * + * @param expectedFetches the fetches expected once it fetched + */ + private void awaitZoneFetch(int expectedFetches) throws InterruptedException { + boolean fetched = waitFor(() -> { + contentModule().onZoneTimerTick(); + return server.contentFetches.size() >= expectedFetches; + }); + Assert.assertTrue("the content zone never fetched, logs: " + logs, fetched); + Assert.assertEquals(expectedFetches, server.contentFetches.size()); + } + + /** + * Waits until the log holds a number of lines containing a text. + * + * @param text the text + * @param count how many lines + */ + private void awaitLogs(String text, int count) throws InterruptedException { + Assert.assertTrue("never logged [" + text + "] " + count + " times", waitFor(() -> logsContaining(text) >= count)); + Assert.assertEquals(count, logsContaining(text)); + } + + /** + * Waits until every stored request was sent and removed. + */ + private void awaitEmptyQueue() throws InterruptedException { + Assert.assertTrue("the request queue never drained", waitFor(() -> queuedRequestCount() == 0)); + } + + /** + * How many log lines so far contain a text. + * + * @param text the text + * @return the number of lines + */ + private int logsContaining(String text) { + int count = 0; + for (String line : logs) { + if (line.contains(text)) { + count++; + } + } + return count; + } + + /** + * Counts the stored requests without reading them, as the network loop can remove one while it + * is read. + * + * @return the number of stored requests + */ + private static int queuedRequestCount() { + File[] files = TestUtils.getTestSDirectory().listFiles((dir, name) -> name.startsWith("[CLY]_request_")); + return files == null ? 0 : files.length; + } + + /** + * The storage IDs of the stored requests. + * + * @return the IDs + */ + private static Set storedRequestIds() { + Set ids = new HashSet<>(); + File[] files = TestUtils.getTestSDirectory().listFiles((dir, name) -> name.startsWith("[CLY]_request_")); + if (files != null) { + for (File file : files) { + ids.add(Long.parseLong(file.getName().substring("[CLY]_request_".length()))); + } + } + return ids; + } + + /** + * The keys of the events a request carries, in order. + * + * @param params the parameters of the request + * @return the keys as a JSON array + */ + private static JSONArray eventKeys(Map params) { + JSONArray events = new JSONArray(params.get("events")); + JSONArray keys = new JSONArray(); + for (int i = 0; i < events.length(); i++) { + keys.put(events.getJSONObject(i).getString("key")); + } + return keys; + } + + /** + * Polls until the condition holds or {@link #WAIT_BUDGET_MS} runs out, nudging the send loop on + * every round, as it only picks up a request when something is pushed. + * + * @param condition the condition + * @return whether the condition holds + */ + private static boolean waitFor(BooleanSupplier condition) throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_BUDGET_MS; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return true; + } + SDKCore core = SDKCore.instance; + if (core != null && core.networking != null) { + core.networking.check(core.config); + } + Thread.sleep(50); + } + return condition.getAsBoolean(); + } + + /** + * Decodes url encoded request parameters. + * + * @param data the parameters + * @return the parameters by name + */ + private static Map parseParams(String data) { + Map params = new HashMap<>(); + for (String pair : data.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] keyValue = pair.split("=", 2); + params.put(Utils.urldecode(keyValue[0]), keyValue.length > 1 ? Utils.urldecode(keyValue[1]) : ""); + } + return params; + } + + /** + * Reads a stream to its end. + * + * @param stream the stream + * @return the bytes + * @throws IOException when the stream cannot be read + */ + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = stream.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationFiltersTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationFiltersTests.java new file mode 100644 index 00000000..ba7773d1 --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationFiltersTests.java @@ -0,0 +1,447 @@ +package ly.count.sdk.java.internal; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import ly.count.sdk.java.PredefinedUserPropertyKeys; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * The listing filters and the user property cache limit of the SDK behavior settings, observed + * through the public API: which custom events, segmentation keys and custom user properties reach + * the event queue and the request queue, while internal events and predefined user properties pass. + *

+ * Settings requests go through a {@link ModuleConfigurationTests.ServerConfigResponder}, which answers + * on the calling thread, so the response of the fetch made at init applies before init returns and a + * later one is pushed by fetching the way the refresh timer does. Networking is off through the + * settings, so every stored request stays on disk to be read. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationFiltersTests { + + private ModuleConfigurationTests.ServerConfigResponder server; + private final List warnings = new CopyOnWriteArrayList<>(); + + /** + * Starts every test from empty storage, without a test module and with a fresh responder. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new ModuleConfigurationTests.ServerConfigResponder(); + warnings.clear(); + } + + /** + * Stops the SDK and clears its data. + */ + @After + public void afterTest() { + Countly.instance().halt(); + } + + // region scenarios + + /** + * {@code eb} and then {@code ew} decide which custom events are recorded, while internal events + * always pass. + *

+ * Verifies that with a blacklist, listed keys, spaces and dashes and a {@code [CLY]_} key the SDK + * does not record itself included, are dropped with one warning each, timed events too, while an + * unlisted key, a view and an internal event listed in the blacklist are recorded; that a response bringing a whitelist replaces the blacklist, so only + * its keys are recorded, a view still is and a key the blacklist dropped is dropped again; and that + * an empty whitelist allows every key. + */ + @Test + public void eventFilter_blacklistThenWhitelist_decideCustomEventsOnly_andAnEmptyListAllowsEverything() { + server.respondWith(new ServerConfigBuilder() + .eventFilterList(names("blocked_event", "event with spaces", "event-with-dashes", "[CLY]_not_an_sdk_event", ModuleViews.KEY_VIEW_EVENT, "[CLY]_star_rating"), false) + .networking(false)); + init(configWithWarnings().enableFeatures(Config.Feature.Events, Config.Feature.Views).setEventQueueSizeToSend(100)); + + Countly.instance().events().recordEvent("blocked_event"); + Countly.instance().events().recordEvent("event with spaces"); + Countly.instance().events().recordEvent("event-with-dashes", TestUtils.map("colour", "red")); + Countly.instance().events().recordEvent("[CLY]_not_an_sdk_event"); + Countly.instance().events().startEvent("blocked_event"); + Countly.instance().events().endEvent("blocked_event"); + TestUtils.validateEQSize(0); + Assert.assertEquals(5, warningsContaining("is filtered out by the event filter")); + + Countly.instance().events().recordEvent("allowed_event"); + Countly.instance().views().startView("home"); + Countly.instance().events().recordEvent("[CLY]_star_rating"); + Assert.assertEquals(Arrays.asList("allowed_event", ModuleViews.KEY_VIEW_EVENT, "[CLY]_star_rating"), TestUtils.keysInEQ()); + + push(new ServerConfigBuilder().eventFilterList(names("allowed_event", "another_allowed"), true)); + Countly.instance().events().recordEvent("allowed_event"); + Countly.instance().events().recordEvent("not_in_whitelist"); + Countly.instance().events().recordEvent("another_allowed", 2); + Countly.instance().events().recordEvent("blocked_event"); + Countly.instance().views().stopViewWithName("home"); + Assert.assertEquals(Arrays.asList("allowed_event", ModuleViews.KEY_VIEW_EVENT, "[CLY]_star_rating", + "allowed_event", "another_allowed", ModuleViews.KEY_VIEW_EVENT), TestUtils.keysInEQ()); + Assert.assertEquals(7, warningsContaining("is filtered out by the event filter")); + + push(new ServerConfigBuilder().eventFilterList(names(), true)); + Countly.instance().events().recordEvent("not_in_whitelist"); + Countly.instance().events().recordEvent("blocked_event"); + List keys = TestUtils.keysInEQ(); + Assert.assertEquals(Arrays.asList("not_in_whitelist", "blocked_event"), keys.subList(6, keys.size())); + Assert.assertEquals(7, warningsContaining("is filtered out by the event filter")); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + } + + /** + * {@code sb} and {@code esb}, then {@code sw} and {@code esw}, remove segmentation keys of custom + * events on a copy of the map the caller passed. + *

+ * Verifies on the wire that the segmentation blacklist and then the blacklist of each event apply + * together, that an event without rules keeps everything but the general blacklist, that the + * caller's unmodifiable map is left as it was and can be passed again, that an internal event and + * a view keep every key, that empty and missing segmentation still record, and that a response + * bringing whitelists replaces both blacklists, the general whitelist applying before the one of + * the event. + */ + @Test + public void segmentationFilters_generalThenPerEvent_filterACopyOfTheCallersMap_andNeverTouchInternalEvents() { + Map> perEvent = new LinkedHashMap<>(); + perEvent.put("event1", names("blocked_for_event1")); + perEvent.put("event2", names("key_b")); + server.respondWith(new ServerConfigBuilder() + .segmentationFilterList(names("general_blocked"), false) + .eventSegmentationFilterMap(perEvent, false) + .eventQueueSize(1) + .networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events, Config.Feature.Views)); + + Map callers = Collections.unmodifiableMap(TestUtils.map("general_blocked", "g", "blocked_for_event1", "e1", "key_a", "a", "key_b", "b")); + Countly.instance().events().recordEvent("event1", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event2", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event3", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("[CLY]_star_rating", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().views().startView("home", TestUtils.map("general_blocked", "v")); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event1", new HashMap<>()); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event1"); + TestUtils.letTheClockCatchUp(); + + List sent = sentEvents(); + Assert.assertEquals(keysOf(sent).toString(), 7, sent.size()); + assertEvent(sent.get(0), "event1", TestUtils.map("key_a", "a", "key_b", "b")); + assertEvent(sent.get(1), "event2", TestUtils.map("blocked_for_event1", "e1", "key_a", "a")); + assertEvent(sent.get(2), "event3", TestUtils.map("blocked_for_event1", "e1", "key_a", "a", "key_b", "b")); + assertEvent(sent.get(3), "[CLY]_star_rating", callers); + Assert.assertEquals(ModuleViews.KEY_VIEW_EVENT, sent.get(4).key); + Assert.assertEquals("v", sent.get(4).segmentation.get("general_blocked")); + Assert.assertEquals("home", sent.get(4).segmentation.get(ModuleViews.KEY_NAME)); + Assert.assertEquals("event1", sent.get(5).key); + Assert.assertTrue(sent.get(5).segmentation == null || sent.get(5).segmentation.isEmpty()); + Assert.assertEquals("event1", sent.get(6).key); + Assert.assertTrue(sent.get(6).segmentation == null || sent.get(6).segmentation.isEmpty()); + Assert.assertEquals(TestUtils.map("general_blocked", "g", "blocked_for_event1", "e1", "key_a", "a", "key_b", "b"), callers); + + Map> perEventWhitelist = new LinkedHashMap<>(); + perEventWhitelist.put("event1", names("key_a")); + push(new ServerConfigBuilder() + .segmentationFilterList(names("key_a", "key_b", "blocked_for_event1"), true) + .eventSegmentationFilterMap(perEventWhitelist, true) + .eventQueueSize(1) + .networking(false)); + Countly.instance().events().recordEvent("event1", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event2", callers); + TestUtils.letTheClockCatchUp(); + Countly.instance().events().recordEvent("event4", TestUtils.map("key_b", "b", "other", "o")); + TestUtils.letTheClockCatchUp(); + + sent = sentEvents(); + Assert.assertEquals(keysOf(sent).toString(), 10, sent.size()); + assertEvent(sent.get(7), "event1", TestUtils.map("key_a", "a")); + assertEvent(sent.get(8), "event2", TestUtils.map("blocked_for_event1", "e1", "key_a", "a", "key_b", "b")); + assertEvent(sent.get(9), "event4", TestUtils.map("key_b", "b")); + Assert.assertEquals(4, callers.size()); + } + + /** + * {@code upb} and then {@code upw} decide which custom user properties are set and modified, + * while predefined properties are never filtered. + *

+ * Verifies on the wire that with a blacklist listing custom keys and predefined ones, the + * predefined name and email are still sent while a listed custom key is dropped from a batch, + * from a single set and from each of the nine modifications, with one warning per dropped call, + * and every unlisted key keeps its set or modification; and that a response bringing a whitelist + * keeps only its custom keys while a predefined key outside it is still sent. + */ + @Test + public void userPropertyFilter_appliesToCustomKeysOfEverySetAndModification_whilePredefinedKeysBypassIt() { + server.respondWith(new ServerConfigBuilder() + .userPropertyFilterList(names("blocked_prop", "blocked_counter", PredefinedUserPropertyKeys.NAME, PredefinedUserPropertyKeys.EMAIL), false) + .networking(false)); + init(configWithWarnings()); + + Map batch = new LinkedHashMap<>(); + batch.put(PredefinedUserPropertyKeys.NAME, "John"); + batch.put(PredefinedUserPropertyKeys.EMAIL, "john@example.com"); + batch.put("blocked_prop", "value1"); + batch.put("allowed_prop", "value2"); + Countly.instance().userProfile().setProperties(Collections.unmodifiableMap(batch)); + Countly.instance().userProfile().setProperty("blocked_prop", "value3"); + modifyWithEveryOperation("blocked_counter"); + modifyWithEveryOperation("allowed_counter"); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + JSONObject details = userDetails(0); + Assert.assertEquals("John", details.getString(PredefinedUserPropertyKeys.NAME)); + Assert.assertEquals("john@example.com", details.getString(PredefinedUserPropertyKeys.EMAIL)); + JSONObject custom = details.getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(names("allowed_prop", "allowed_counter"), custom.keySet()); + Assert.assertEquals("value2", custom.getString("allowed_prop")); + JSONObject counter = custom.getJSONObject("allowed_counter"); + Assert.assertEquals(names("$inc", "$mul", "$max", "$min", "$setOnce", "$push", "$addToSet", "$pull"), counter.keySet()); + Assert.assertEquals(5, counter.getInt("$inc")); + Assert.assertEquals(11, warningsContaining("is filtered out by the user property filter")); + + push(new ServerConfigBuilder().userPropertyFilterList(names("plan", "visits"), true)); + Countly.instance().userProfile().setProperty("plan", "gold"); + Countly.instance().userProfile().setProperty("allowed_prop", "not_listed_now"); + Countly.instance().userProfile().setProperty(PredefinedUserPropertyKeys.USERNAME, "jdoe"); + Countly.instance().userProfile().increment("visits"); + Countly.instance().userProfile().increment("allowed_counter"); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + details = userDetails(1); + Assert.assertEquals("jdoe", details.getString(PredefinedUserPropertyKeys.USERNAME)); + custom = details.getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(names("plan", "visits"), custom.keySet()); + Assert.assertEquals("gold", custom.getString("plan")); + Assert.assertEquals(1, custom.getJSONObject("visits").getInt("$inc")); + Assert.assertEquals(13, warningsContaining("is filtered out by the user property filter")); + } + + /** + * {@code upcl} caps the pending custom properties and the keys with pending modifications apart. + *

+ * Verifies that without the setting any number of custom properties is sent; that with it, a + * batch over the limit keeps its newest keys, predefined properties are neither counted nor + * dropped, updating a pending key does not count again, a new key drops the oldest one, and each + * drop warns; that modifications count distinct keys, so repeating one does not count, and going + * over the limit drops every modification of the oldest key; that the request carries exactly + * what was kept; and that a save starts the count again. + */ + @Test + public void userPropertyCacheLimit_capsPendingCustomSetsAndModifiedKeysApart_droppingTheOldestFirst() { + server.respondWith(new ServerConfigBuilder().networking(false)); + init(configWithWarnings()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider().getUserPropertyCacheLimit()); + + for (int i = 1; i <= 150; i++) { + Countly.instance().userProfile().setProperty("bulk_" + i, i); + } + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + Assert.assertEquals(150, userDetails(0).getJSONObject(ModuleUserProfile.CUSTOM_KEY).length()); + + push(new ServerConfigBuilder().userPropertyCacheLimit(3)); + Map batch = new LinkedHashMap<>(); + for (int i = 1; i <= 5; i++) { + batch.put("prop" + i, "value" + i); + } + batch.put(PredefinedUserPropertyKeys.NAME, "John"); + batch.put(PredefinedUserPropertyKeys.EMAIL, "john@example.com"); + batch.put(PredefinedUserPropertyKeys.USERNAME, "jdoe"); + Countly.instance().userProfile().setProperties(batch); + Assert.assertEquals(1, warningsContaining("over the cache limit of [3]")); + Countly.instance().userProfile().setProperty("prop3", "updated"); + Assert.assertEquals(1, warningsContaining("over the cache limit of [3]")); + Countly.instance().userProfile().setProperty("prop6", "value6"); + Assert.assertEquals(2, warningsContaining("over the cache limit of [3]")); + + Countly.instance().userProfile().incrementBy("counter1", 1); + Countly.instance().userProfile().incrementBy("counter2", 2); + Countly.instance().userProfile().incrementBy("counter1", 10); + Countly.instance().userProfile().incrementBy("counter3", 3); + Assert.assertEquals(2, warningsContaining("over the cache limit of [3]")); + Countly.instance().userProfile().multiply("factor", 2); + Countly.instance().userProfile().push("list", "x"); + Assert.assertEquals(4, warningsContaining("over the cache limit of [3]")); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + JSONObject details = userDetails(1); + Assert.assertEquals("John", details.getString(PredefinedUserPropertyKeys.NAME)); + Assert.assertEquals("john@example.com", details.getString(PredefinedUserPropertyKeys.EMAIL)); + Assert.assertEquals("jdoe", details.getString(PredefinedUserPropertyKeys.USERNAME)); + JSONObject custom = details.getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(names("prop4", "prop5", "prop6", "counter3", "factor", "list"), custom.keySet()); + Assert.assertEquals("value4", custom.getString("prop4")); + Assert.assertEquals(3, custom.getJSONObject("counter3").getInt("$inc")); + Assert.assertEquals(2.0, custom.getJSONObject("factor").getDouble("$mul"), 0.0); + Assert.assertEquals("x", custom.getJSONObject("list").getString("$push")); + + Countly.instance().userProfile().setProperties(TestUtils.map("fresh1", "a", "fresh2", "b", "fresh3", "c")); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + Assert.assertEquals(3, userDetails(2).getJSONObject(ModuleUserProfile.CUSTOM_KEY).length()); + Assert.assertEquals(4, warningsContaining("over the cache limit of [3]")); + } + + // endregion + // region helpers + + /** + * Initializes the SDK with {@link #server} answering its settings requests. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + server.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * The settings test configuration with a listener that keeps every warning. + * + * @return the configuration + */ + private Config configWithWarnings() { + return TestUtils.getConfigSdkBehaviorSettings().setLogListener((message, level) -> { + if (level == Config.LoggingLevel.WARN) { + warnings.add(message); + } + }); + } + + /** + * How many warnings so far contain a text. + * + * @param text the text + * @return the number of warnings + */ + private int warningsContaining(String text) { + int count = 0; + for (String warning : warnings) { + if (warning.contains(text)) { + count++; + } + } + return count; + } + + /** + * Every event of every stored request, in queue order. + * + * @return the events + */ + private static List sentEvents() { + List events = new ArrayList<>(); + int requestCount = TestUtils.getCurrentRQ().length; + for (int i = 0; i < requestCount; i++) { + events.addAll(TestUtils.readEventsFromRequest(i, TestUtils.DEVICE_ID)); + } + return events; + } + + /** + * The keys of events, in order. + * + * @param events the events + * @return the keys + */ + private static List keysOf(List events) { + List keys = new ArrayList<>(); + for (EventImpl event : events) { + keys.add(event.key); + } + return keys; + } + + /** + * Asserts the key and the whole segmentation of an event. + * + * @param event the event + * @param key the expected key + * @param segmentation the expected segmentation + */ + private static void assertEvent(EventImpl event, String key, Map segmentation) { + Assert.assertEquals(key, event.key); + Assert.assertEquals(segmentation, new HashMap<>(event.segmentation)); + } + + /** + * The {@code user_details} object of a stored request. + * + * @param requestIndex the index of the request in the queue + * @return the user details + */ + private static JSONObject userDetails(int requestIndex) { + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertTrue("no request at index " + requestIndex + " in " + Arrays.toString(requests), requests.length > requestIndex); + return new JSONObject(requests[requestIndex].get("user_details")); + } + + /** + * Applies each of the nine modifications of the user profile interface to one key. + * + * @param key the custom user property key + */ + private static void modifyWithEveryOperation(String key) { + ModuleUserProfile.UserProfile profile = Countly.instance().userProfile(); + profile.increment(key); + profile.incrementBy(key, 4); + profile.multiply(key, 3); + profile.saveMax(key, 10); + profile.saveMin(key, 1); + profile.setOnce(key, "first"); + profile.push(key, "a"); + profile.pushUnique(key, "b"); + profile.pull(key, "c"); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationGatesTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationGatesTests.java new file mode 100644 index 00000000..153aa39d --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationGatesTests.java @@ -0,0 +1,876 @@ +package ly.count.sdk.java.internal; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import ly.count.sdk.java.PredefinedUserPropertyKeys; +import ly.count.sdk.java.View; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * The switches and tunables of the SDK behavior settings, observed through the public API: what each + * one lets into the request queue and the event queue, both when it comes from the stored or provided + * settings at init and when a server response changes it while the SDK runs. + *

+ * Settings requests go through a {@link ModuleConfigurationTests.ServerConfigResponder}, which answers + * on the calling thread: the response of the fetch made at init is applied before init returns, and a + * later one is pushed by fetching the way the refresh timer does. Most scenarios switch networking off + * through the settings, so every stored request stays on disk to be read. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationGatesTests { + + private ModuleConfigurationTests.ServerConfigResponder server; + private Thread.UncaughtExceptionHandler defaultHandlerBeforeTest; + + /** + * Starts every test from empty storage, without a test module and with a fresh responder. + */ + @Before + public void beforeTest() { + defaultHandlerBeforeTest = Thread.getDefaultUncaughtExceptionHandler(); + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new ModuleConfigurationTests.ServerConfigResponder(); + } + + /** + * Stops the SDK, clearing its data, and puts back the default uncaught exception handler. + */ + @After + public void afterTest() { + Countly.instance().halt(); + SDKCore.testDummyModule = null; + Thread.setDefaultUncaughtExceptionHandler(defaultHandlerBeforeTest); + } + + // region scenarios + + /** + * {@code tracking} off keeps everything new out of storage. + *

+ * Verifies with the switch provided at init that custom and internal events, timed events, views, + * sessions, breadcrumbs, handled and unhandled crashes, location, user properties, an A/B test + * enrollment and a device ID merge store no request and no event and leave no crash file, that a + * crash file left by an earlier run is still turned into a request, and that events are recorded + * again once a response turns tracking back on. + */ + @Test + public void tracking_off_storesNoRequestAndNoEvent_whileACrashOfAnEarlierRunIsStillSent() { + long crashId = TimeUtils.uniqueTimestampMs(); + TestUtils.writeToFile("crash_" + crashId, new JSONObject().put("_error", "earlier_run_crash").put("_nonfatal", false).toString()); + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Events, Config.Feature.Sessions, Config.Feature.Views, Config.Feature.CrashReporting, + Config.Feature.Location, Config.Feature.UserProfiles, Config.Feature.RemoteConfig) + .setEventQueueSizeToSend(1) + .setSdkBehaviorSettings(new ServerConfigBuilder().tracking(false).networking(false).build())); + + Assert.assertFalse(provider().getTrackingEnabled()); + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(1, requests.length); + Assert.assertTrue(requests[0].get("crash").contains("earlier_run_crash")); + Assert.assertEquals(0, filesWithPrefix("[CLY]_crash_")); + + Countly.instance().events().recordEvent("custom_event"); + Countly.instance().events().recordEvent("[CLY]_star_rating"); + Countly.instance().events().startEvent("timed_event"); + Countly.instance().events().endEvent("timed_event"); + Assert.assertNull(Countly.instance().views().startView("home")); + Countly.session().begin(); + Countly.session().update(); + Countly.session().end(); + Countly.instance().crashes().addCrashBreadcrumb("breadcrumb"); + Countly.instance().crashes().recordHandledException(new Exception("handled_while_tracking_is_off")); + Countly.instance().crashes().recordUnhandledException(new Exception("unhandled_while_tracking_is_off")); + Countly.instance().location().setLocation("TR", "Izmir", "1,2", "1.1.1.1"); + Countly.instance().location().disableLocation(); + Countly.instance().userProfile().setProperty("plan", "gold"); + Countly.instance().userProfile().save(); + Countly.instance().remoteConfig().enrollIntoABTestsForKeys(new String[] { "button_color" }); + Countly.instance().deviceId().changeWithMerge("merged_device_id"); + + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + TestUtils.validateEQSize(0); + Assert.assertEquals(0, filesWithPrefix("[CLY]_crash_")); + Assert.assertEquals(0, filesWithPrefix("[CLY]_session_")); + + push(new ServerConfigBuilder().tracking(true)); + Countly.instance().events().recordEvent("custom_event_after"); + + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(2, requests.length); + Assert.assertEquals("merged_device_id", requests[1].get("device_id")); + Assert.assertEquals("custom_event_after", new JSONArray(requests[1].get("events")).getJSONObject(0).getString("key")); + } + + /** + * {@code st} off ignores beginning, updating and ending a session, whatever calls them. + *

+ * Verifies that after a response turns session tracking off, an update, an end, a timer tick and a + * device ID change without merge send no session request while the running session keeps its + * state, that the session file left by that run is removed at the next init without an end + * request, that a session neither begins through the public API nor through a device ID change + * while the stored switch is off, that consent to sessions begins nothing either, and that a + * response turning it back on lets a session begin. + */ + @Test + public void sessionTracking_off_sendsNoSessionRequest_fromAnyCaller_andRecoveryDropsTheLeftoverSession() { + server.respondWith(new ServerConfigBuilder().networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Sessions)); + + Countly.session().begin(); + Assert.assertEquals(1, requestsWith("begin_session")); + + push(new ServerConfigBuilder().sessionTracking(false)); + Countly.session().update(); + Countly.session().end(); + SDKCore.instance.module(ModuleSessions.class).onTimer(); + Countly.instance().deviceId().changeWithoutMerge("device_id_2"); + + Assert.assertTrue(Countly.session().isActive()); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + Assert.assertEquals(0, requestsWith("session_duration") + requestsWith("end_session")); + Countly.instance().stop(); + Assert.assertEquals(1, filesWithPrefix("[CLY]_session_")); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Sessions)); + + Assert.assertFalse(provider().getSessionTrackingEnabled()); + Assert.assertEquals(0, filesWithPrefix("[CLY]_session_")); + SessionImpl session = (SessionImpl) Countly.session(); + session.begin(); + Assert.assertFalse(session.isActive()); + Assert.assertNull(session.update(null)); + Assert.assertNull(session.end(null, null, null)); + Countly.instance().deviceId().changeWithoutMerge("device_id_3"); + Assert.assertFalse(Countly.session().isActive()); + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(1, requests.length); + Assert.assertEquals("1", requests[0].get("begin_session")); + Countly.instance().halt(); + + init(TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Sessions) + .setRequiresConsent(true) + .setSdkBehaviorSettings(new ServerConfigBuilder().sessionTracking(false).networking(false).build())); + Countly.onConsent(Config.Feature.Sessions); + Assert.assertTrue(Countly.isTracking(Config.Feature.Sessions)); + Assert.assertFalse(Countly.session().isActive()); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + + push(new ServerConfigBuilder().sessionTracking(true)); + Countly.session().begin(); + Assert.assertTrue(Countly.session().isActive()); + Assert.assertEquals(1, requestsWith("begin_session")); + } + + /** + * {@code vt} off ignores every view call, while custom events keep being recorded. + *

+ * Verifies that after a response turns view tracking off, starting, stopping by name and by ID, + * pausing, resuming and stopping all views record no {@code [CLY]_view} event, through the views + * interface and through the deprecated session views alike, that a custom event is still + * recorded, that the views keep running and end once a response turns view tracking back on, + * and that the stored switch keeps views off after a restart. + */ + @Test + public void viewTracking_off_ignoresEveryViewCall_whileCustomEventsStillRecord() { + Config config = TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Events, Config.Feature.Views, Config.Feature.Sessions) + .setEventQueueSizeToSend(100); + server.respondWith(new ServerConfigBuilder().networking(false)); + init(config); + + String homeId = Countly.instance().views().startView("home"); + String settingsId = Countly.instance().views().startView("settings"); + Assert.assertEquals(2, viewEvents().size()); + + push(new ServerConfigBuilder().viewTracking(false)); + Assert.assertNull(Countly.instance().views().startView("blocked")); + Assert.assertNull(Countly.instance().views().startAutoStoppedView("blocked_auto_stopped")); + Countly.instance().views().stopViewWithName("home"); + Countly.instance().views().stopViewWithID(settingsId); + Countly.instance().views().pauseViewWithID(homeId); + Countly.instance().views().resumeViewWithID(homeId); + Countly.instance().views().stopAllViews(null); + View legacyView = ((SessionImpl) Countly.session()).view("legacy_view"); + legacyView.stop(false); + Countly.instance().events().recordEvent("custom_event"); + + List events = TestUtils.getCurrentEQ(); + Assert.assertEquals(3, events.size()); + Assert.assertEquals("custom_event", events.get(2).key); + Assert.assertEquals(2, viewEvents().size()); + + push(new ServerConfigBuilder().viewTracking(true)); + Countly.instance().views().stopViewWithID(homeId); + Countly.instance().views().stopAllViews(null); + + List views = viewEvents(); + Assert.assertEquals(4, views.size()); + Assert.assertEquals("home", views.get(2).segmentation.get("name")); + Assert.assertEquals("settings", views.get(3).segmentation.get("name")); + Assert.assertNull(views.get(2).segmentation.get("visit")); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + + push(new ServerConfigBuilder().viewTracking(false)); + Countly.instance().stop(); + server.pending(); + init(config); + + Assert.assertFalse(provider().getViewTrackingEnabled()); + Assert.assertNull(Countly.instance().views().startView("after_restart")); + Assert.assertEquals(4, viewEvents().size()); + Assert.assertEquals(5, TestUtils.getCurrentEQ().size()); + } + + /** + * {@code crt} and {@code acr} decide whether unhandled crashes are caught, a response can install + * the handler while the SDK runs, and handled exceptions follow {@code crt} alone. + *

+ * Verifies that the handler is not installed while {@code acr} is off, that a response turning it + * on installs it, that a later change never installs it twice, that it records an unhandled crash + * only while both switches are on at crash time and always hands the crash to the handler it + * replaced, that handled and unhandled exceptions reported through the crashes interface are + * recorded whenever {@code crt} is on, and that stopping the SDK leaves in place a handler the + * application installed over it, which then only reaches the application's handler. + */ + @Test + public void crashSwitches_decideWhatIsRecorded_andAResponseInstallsTheHandlerOnce() throws InterruptedException { + List reachedTheApp = new CopyOnWriteArrayList<>(); + Thread.UncaughtExceptionHandler appHandler = (thread, throwable) -> reachedTheApp.add(throwable); + Thread.setDefaultUncaughtExceptionHandler(appHandler); + List crashesBeingRecorded = new CopyOnWriteArrayList<>(); + + server.respondWith(new ServerConfigBuilder().automaticCrashReporting(false).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting).setLogListener((message, level) -> { + if (message.startsWith("[ModuleCrash] onCrash:")) { + crashesBeingRecorded.add(message); + } + })); + + Assert.assertTrue(provider().getCrashReportingEnabled()); + Assert.assertFalse(provider().getAutomaticCrashReportingEnabled()); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + crashOnAThread(new RuntimeException("crash_while_acr_is_off")); + Assert.assertEquals(1, reachedTheApp.size()); + Assert.assertEquals(0, crashRequests().size()); + + push(new ServerConfigBuilder().automaticCrashReporting(true)); + Thread.UncaughtExceptionHandler installed = Thread.getDefaultUncaughtExceptionHandler(); + Assert.assertNotSame(appHandler, installed); + push(new ServerConfigBuilder().automaticCrashReporting(true).eventQueueSize(5)); + Assert.assertSame(installed, Thread.getDefaultUncaughtExceptionHandler()); + + crashOnAThread(new RuntimeException("crash_while_both_are_on")); + Assert.assertEquals(2, reachedTheApp.size()); + List crashes = crashRequests(); + Assert.assertEquals(1, crashes.size()); + Assert.assertTrue(crashes.get(0).getString("_error").contains("crash_while_both_are_on")); + Assert.assertFalse(crashes.get(0).getBoolean("_nonfatal")); + + push(new ServerConfigBuilder().crashReporting(false)); + crashOnAThread(new RuntimeException("crash_while_crt_is_off")); + Countly.instance().crashes().recordHandledException(new Exception("handled_while_crt_is_off")); + Assert.assertEquals(3, reachedTheApp.size()); + Assert.assertEquals(1, crashRequests().size()); + + push(new ServerConfigBuilder().crashReporting(true).automaticCrashReporting(false)); + crashOnAThread(new RuntimeException("crash_while_acr_is_off_again")); + Countly.instance().crashes().recordHandledException(new Exception("handled_while_crt_is_on")); + Countly.instance().crashes().recordUnhandledException(new Exception("reported_while_crt_is_on")); + Assert.assertEquals(4, reachedTheApp.size()); + Assert.assertSame(installed, Thread.getDefaultUncaughtExceptionHandler()); + crashes = crashRequests(); + Assert.assertEquals(3, crashes.size()); + Assert.assertTrue(crashes.get(1).getString("_error").contains("handled_while_crt_is_on")); + Assert.assertTrue(crashes.get(1).getBoolean("_nonfatal")); + Assert.assertTrue(crashes.get(2).getString("_error").contains("reported_while_crt_is_on")); + Assert.assertFalse(crashes.get(2).getBoolean("_nonfatal")); + Assert.assertEquals(3, crashesBeingRecorded.size()); + + List reachedTheLaterHandler = new CopyOnWriteArrayList<>(); + Thread.UncaughtExceptionHandler laterHandler = (thread, throwable) -> { + reachedTheLaterHandler.add(throwable); + installed.uncaughtException(thread, throwable); + }; + Thread.setDefaultUncaughtExceptionHandler(laterHandler); + push(new ServerConfigBuilder().automaticCrashReporting(true)); + Countly.instance().halt(); + + Assert.assertSame(laterHandler, Thread.getDefaultUncaughtExceptionHandler()); + crashOnAThread(new RuntimeException("crash_after_halt")); + Assert.assertEquals(1, reachedTheLaterHandler.size()); + Assert.assertEquals(5, reachedTheApp.size()); + Assert.assertEquals(3, crashesBeingRecorded.size()); + Assert.assertEquals(0, filesWithPrefix("[CLY]_crash_")); + Assert.assertEquals(0, filesWithPrefix("[CLY]_request_")); + } + + /** + * The handler installed at init follows the resolved {@code crt} and {@code acr} and crash consent. + *

+ * Verifies that the developer default installs it, that a developer who disabled unhandled crash + * reporting gets none unless the response of the fetch made at init turns {@code acr} on, which + * installs it once even though the response arrives before the crash module finished its init, + * that stored {@code crt} off keeps it out until a response turns {@code crt} on, that it waits + * for crash consent, and that each stop puts the application's handler back. + */ + @Test + public void crashHandler_atInit_followsTheResolvedSwitchesAndConsent() { + Thread.UncaughtExceptionHandler appHandler = (thread, throwable) -> { + }; + Thread.setDefaultUncaughtExceptionHandler(appHandler); + + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting)); + Assert.assertNotSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.instance().halt(); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting).disableUnhandledCrashReporting()); + Assert.assertFalse(provider().getAutomaticCrashReportingEnabled()); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.instance().halt(); + + server.respondWith(new ServerConfigBuilder().automaticCrashReporting(true)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting).disableUnhandledCrashReporting()); + Assert.assertTrue(provider().getAutomaticCrashReportingEnabled()); + Assert.assertNotSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.instance().halt(); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.CrashReporting) + .setSdkBehaviorSettings(new ServerConfigBuilder().crashReporting(false).build())); + Assert.assertTrue(provider().getAutomaticCrashReportingEnabled()); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + push(new ServerConfigBuilder().crashReporting(true)); + Assert.assertNotSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.instance().stop(); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting).setRequiresConsent(true)); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.onConsent(Config.Feature.CrashReporting); + Assert.assertNotSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + Countly.instance().halt(); + Assert.assertSame(appHandler, Thread.getDefaultUncaughtExceptionHandler()); + } + + /** + * {@code lt} off keeps location out of every request, and turning it off erases the location + * stored on the server. + *

+ * Verifies that with location tracking on, a location is sent on its own and with the begin + * request; that a response turning it off sends one erase request and a later change sends no + * second one; that while it is off setting a location through the location interface, the + * deprecated session call and the deprecated request builder sends nothing and a begin request + * carries no location; that while session tracking is off a location is not held back for a + * session that will not begin; that a response at init turning it off while the developer + * disabled location sends a single erase request; and that the stored switch keeps the + * developer's location out of the next init. + */ + @Test + public void locationTracking_off_keepsLocationOutOfEveryRequest_andTurningItOffErasesIt() { + Config config = TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Location, Config.Feature.Sessions); + server.respondWith(new ServerConfigBuilder().networking(false)); + init(config); + + Countly.instance().location().setLocation("TR", "Izmir", "1,2", "1.1.1.1"); + Countly.session().begin(); + Countly.session().end(); + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(3, requests.length); + Assert.assertEquals("Izmir", requests[0].get("city")); + Assert.assertEquals("1", requests[1].get("begin_session")); + Assert.assertEquals("Izmir", requests[1].get("city")); + Assert.assertEquals("1.1.1.1", requests[1].get("ip")); + + push(new ServerConfigBuilder().locationTracking(false)); + push(new ServerConfigBuilder().locationTracking(false).eventQueueSize(5)); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(4, requests.length); + Assert.assertEquals("", requests[3].get("location")); + Assert.assertFalse(requests[3].containsKey("city")); + + Countly.instance().location().setLocation("US", "New York", "3,4", "2.2.2.2"); + Countly.session().addLocation(5.0, 6.0); + Assert.assertNull(ModuleRequests.location(SDKCore.instance.config, 7.0, 8.0)); + Countly.session().begin(); + Countly.session().end(); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(Arrays.toString(requests), 6, requests.length); + Assert.assertEquals("1", requests[4].get("begin_session")); + assertNoLocation(requests[4]); + + push(new ServerConfigBuilder().locationTracking(true).sessionTracking(false)); + Assert.assertNotNull(Countly.session()); + Countly.instance().location().setLocation("NL", "Amsterdam", null, null); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(7, requests.length); + Assert.assertEquals("Amsterdam", requests[6].get("city")); + Countly.instance().halt(); + + server.respondWith(new ServerConfigBuilder().locationTracking(false).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Location, Config.Feature.Sessions).disableLocation()); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(1, requests.length); + Assert.assertEquals("", requests[0].get("location")); + Countly.instance().stop(); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Location, Config.Feature.Sessions).setLocation("DE", "Berlin", "9,9", null)); + Assert.assertFalse(provider().getLocationTrackingEnabled()); + Countly.session().begin(); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(2, requests.length); + Assert.assertEquals("1", requests[1].get("begin_session")); + assertNoLocation(requests[1]); + } + + /** + * {@code lt} off keeps the location of the deprecated user editor out of the user details. + *

+ * Verifies that with location tracking on, a location committed through the user editor is + * sent; that after a response turning it off sends the erase request, neither a location, + * country and city committed through the user editor nor a country and city held from before + * are sent with the user details; that opting out of location services still sends its null + * values; that a country and city held while it is on are not sent once it is turned off and on + * again; and that a location committed then is sent. + */ + @Test + public void locationTracking_off_keepsTheUserEditorLocationOutOfUserDetails() { + server.respondWith(new ServerConfigBuilder().networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Location, Config.Feature.UserProfiles)); + + Countly.instance().user().edit().setLocation(1.5, 2.5).commit(); + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(1, requests.length); + Assert.assertEquals("1.5,2.5", requests[0].get("location")); + + Countly.instance().user().edit().setCountry("TR").setCity("Izmir"); + push(new ServerConfigBuilder().locationTracking(false)); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(2, requests.length); + Assert.assertEquals("", requests[1].get("location")); + + Countly.instance().user().edit().setLocation(56.4, 123.4).setCountry("US").setCity("New York").set(PredefinedUserPropertyKeys.NAME, "Name").commit(); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(3, requests.length); + Assert.assertEquals("Name", new JSONObject(requests[2].get("user_details")).getString("name")); + assertNoLocation(requests[2]); + + Countly.instance().user().edit().optOutFromLocationServices().commit(); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(4, requests.length); + Assert.assertEquals("null", requests[3].get("location")); + Assert.assertEquals("null", requests[3].get("city")); + Assert.assertEquals("null", requests[3].get("country_code")); + + push(new ServerConfigBuilder().locationTracking(true)); + Countly.instance().user().edit().setCountry("NL").setCity("Amsterdam"); + push(new ServerConfigBuilder().locationTracking(false)); + push(new ServerConfigBuilder().locationTracking(true)); + Countly.instance().user().edit().set(PredefinedUserPropertyKeys.NAME, "Other").commit(); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(5, requests.length); + Assert.assertEquals("Other", new JSONObject(requests[4].get("user_details")).getString("name")); + assertNoLocation(requests[4]); + + Countly.instance().user().edit().setLocation(3.5, 4.5).commit(); + requests = TestUtils.getCurrentRQ(); + Assert.assertEquals(6, requests.length); + Assert.assertEquals("3.5,4.5", requests[5].get("location")); + } + + /** + * {@code log} decides what the SDK prints, while the log listener hears every line. + *

+ * Verifies that with the developer level off, provided logging on prints every level from init + * and a response turning it off prints nothing; that a stored off silences a developer level of + * warnings until a response turns it on, which prints that level and not every level; and that + * without the setting the developer level prints as it is. The listener hears the lines that are + * not printed. + */ + @Test + public void logging_decidesWhatIsPrinted_whileTheListenerHearsEveryLine() { + List heard = new CopyOnWriteArrayList<>(); + ByteArrayOutputStream printed = new ByteArrayOutputStream(); + PrintStream originalOut = System.out; + System.setOut(new PrintStream(printed, true)); + try { + server.pending(); + init(configWithProbeListener(heard).setSdkBehaviorSettings(new ServerConfigBuilder().logging(true).build())); + Log log = SDKCore.instance.config.getLogger(); + Assert.assertEquals(Config.LoggingLevel.VERBOSE, log.getPrintLevel()); + log.v("probe_verbose_printed"); + Assert.assertTrue(printedText(printed).contains("probe_verbose_printed")); + + push(new ServerConfigBuilder().logging(false)); + Assert.assertEquals(Config.LoggingLevel.OFF, log.getPrintLevel()); + log.e("probe_error_silenced"); + Assert.assertFalse(printedText(printed).contains("probe_error_silenced")); + Assert.assertTrue(heard.contains("probe_error_silenced")); + Countly.instance().stop(); + + server.pending(); + init(configWithProbeListener(heard).setLoggingLevel(Config.LoggingLevel.WARN)); + log = SDKCore.instance.config.getLogger(); + Assert.assertEquals(Config.LoggingLevel.OFF, log.getPrintLevel()); + log.w("probe_warning_silenced"); + Assert.assertFalse(printedText(printed).contains("probe_warning_silenced")); + + push(new ServerConfigBuilder().logging(true)); + Assert.assertEquals(Config.LoggingLevel.WARN, log.getPrintLevel()); + log.d("probe_debug_below_the_level"); + log.w("probe_warning_printed"); + Assert.assertFalse(printedText(printed).contains("probe_debug_below_the_level")); + Assert.assertTrue(printedText(printed).contains("probe_warning_printed")); + Assert.assertTrue(heard.containsAll(Arrays.asList("probe_warning_silenced", "probe_debug_below_the_level", "probe_warning_printed"))); + Countly.instance().halt(); + + server.pending(); + init(configWithProbeListener(heard).setLoggingLevel(Config.LoggingLevel.ERROR)); + Assert.assertEquals(Config.LoggingLevel.ERROR, SDKCore.instance.config.getLogger().getPrintLevel()); + Countly.instance().halt(); + + init(configWithProbeListener(heard)); + Assert.assertEquals(Config.LoggingLevel.OFF, SDKCore.instance.config.getLogger().getPrintLevel()); + } finally { + System.setOut(originalOut); + } + } + + /** + * {@code rqs} caps the stored request queue only when the settings set it. + *

+ * Verifies that the developer request queue size leaves the stored queue unbounded, that once a + * response sets a size, storing a request drops the oldest ones over it with a warning saying how + * many, and that a stored size also holds at the next init for a crash of an earlier run that is + * turned into a request. + */ + @Test + public void requestQueueSize_fromTheSettings_capsTheStoredQueue_andWithoutItTheQueueIsUnbounded() { + List dropWarnings = new CopyOnWriteArrayList<>(); + Config config = TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Events) + .setEventQueueSizeToSend(1) + .setRequestQueueMaxSize(2) + .setLogListener((message, level) -> { + if (level == Config.LoggingLevel.WARN && message.startsWith("[ModuleRequests] dropOldestRequestsOverQueueLimit")) { + dropWarnings.add(message); + } + }); + server.respondWith(new ServerConfigBuilder().networking(false)); + init(config); + + for (int i = 1; i <= 4; i++) { + Countly.instance().events().recordEvent("event_" + i); + letTheClockCatchUp(); + } + Assert.assertFalse(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(Arrays.asList("event_1", "event_2", "event_3", "event_4"), contentOfEachRequest()); + Assert.assertTrue(dropWarnings.isEmpty()); + + push(new ServerConfigBuilder().requestQueueSize(3)); + Countly.instance().events().recordEvent("event_5"); + Assert.assertEquals(Arrays.asList("event_3", "event_4", "event_5"), contentOfEachRequest()); + Assert.assertEquals(1, dropWarnings.size()); + Assert.assertTrue(dropWarnings.get(0), dropWarnings.get(0).contains("size of [3]") && dropWarnings.get(0).contains("dropped the [2] oldest")); + + Countly.instance().events().recordEvent("event_6"); + Assert.assertEquals(Arrays.asList("event_4", "event_5", "event_6"), contentOfEachRequest()); + Assert.assertEquals(2, dropWarnings.size()); + Assert.assertTrue(dropWarnings.get(1), dropWarnings.get(1).contains("dropped the [1] oldest")); + Countly.instance().stop(); + + TestUtils.writeToFile("crash_" + TimeUtils.uniqueTimestampMs(), new JSONObject().put("_error", "earlier_run_crash").put("_nonfatal", false).toString()); + server.pending(); + init(config); + Assert.assertTrue(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(Arrays.asList("event_5", "event_6", "crash"), contentOfEachRequest()); + Assert.assertEquals(3, dropWarnings.size()); + } + + /** + * {@code lbc} is read each time a breadcrumb is added. + *

+ * Verifies that the limit of the response at init overrides the developer's, that the crash + * request carries only the newest breadcrumbs within it, and that after a response shrinks the + * limit while breadcrumbs are kept, the next breadcrumb brings them within the new limit. + */ + @Test + public void breadcrumbLimit_isReadWhenABreadcrumbIsAdded() { + server.respondWith(new ServerConfigBuilder().breadcrumbLimit(3).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.CrashReporting).setMaxBreadcrumbCount(10)); + Assert.assertEquals(3, provider().getMaxBreadcrumbCount()); + + addBreadcrumbs("a", 5); + Countly.instance().crashes().recordHandledException(new Exception("first")); + + push(new ServerConfigBuilder().breadcrumbLimit(5)); + addBreadcrumbs("b", 5); + push(new ServerConfigBuilder().breadcrumbLimit(2)); + Countly.instance().crashes().addCrashBreadcrumb("c1"); + Countly.instance().crashes().recordHandledException(new Exception("second")); + + List crashes = crashRequests(); + Assert.assertEquals(2, crashes.size()); + Assert.assertEquals("a3\na4\na5", crashes.get(0).getString("_logs")); + Assert.assertEquals("b5\nc1", crashes.get(1).getString("_logs")); + } + + /** + * {@code eqs} and {@code sui} changed by a response apply while the SDK runs. + *

+ * Verifies that the event queue is flushed at the new threshold the next time an event is added, + * that a changed session update interval restarts the global timer at that interval, and that a + * response repeating it keeps the running timer. + */ + @Test + public void eventQueueSizeAndSessionUpdateInterval_changedByAResponse_applyWhileTheSdkRuns() { + server.respondWith(new ServerConfigBuilder().eventQueueSize(5).sessionUpdateInterval(30).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events).setEventQueueSizeToSend(100)); + CountlyTimer timerAtInit = SDKCore.instance.countlyTimer; + Assert.assertEquals(30, timerAtInit.getTimerDelaySeconds()); + + Countly.instance().events().recordEvent("e1"); + Countly.instance().events().recordEvent("e2"); + push(new ServerConfigBuilder().eventQueueSize(3)); + TestUtils.validateEQSize(2); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + + Countly.instance().events().recordEvent("e3"); + TestUtils.validateEQSize(0); + Assert.assertEquals(3, TestUtils.readEventsFromRequest(0, TestUtils.DEVICE_ID).size()); + + push(new ServerConfigBuilder().eventQueueSize(1)); + Countly.instance().events().recordEvent("e4"); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + Assert.assertEquals("e4", TestUtils.readEventsFromRequest(1, TestUtils.DEVICE_ID).get(0).key); + + push(new ServerConfigBuilder().sessionUpdateInterval(45)); + CountlyTimer restarted = SDKCore.instance.countlyTimer; + Assert.assertNotSame(timerAtInit, restarted); + Assert.assertEquals(45, restarted.getTimerDelaySeconds()); + + push(new ServerConfigBuilder().sessionUpdateInterval(45).eventQueueSize(2)); + Assert.assertSame(restarted, SDKCore.instance.countlyTimer); + Countly.instance().events().recordEvent("e5"); + TestUtils.validateEQSize(1); + Countly.instance().events().recordEvent("e6"); + TestUtils.validateEQSize(0); + Assert.assertEquals(3, TestUtils.getCurrentRQ().length); + } + + // endregion + // region helpers + + /** + * Initializes the SDK with {@link #server} answering its settings requests. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + letTheClockCatchUp(); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + letTheClockCatchUp(); + server.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * Waits until the clock is past every timestamp handed out so far. A request file is named after + * a {@link TimeUtils#uniqueTimestampMs()} value, which is unique among the last ten values only, + * so a long burst of requests could reuse a name and overwrite a queued request. + */ + private static void letTheClockCatchUp() { + long latest = TimeUtils.uniqueTimestampMs(); + while (System.currentTimeMillis() <= latest) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + /** + * How many stored requests carry a parameter. + * + * @param param the parameter name + * @return the number of requests + */ + private static int requestsWith(String param) { + int count = 0; + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey(param)) { + count++; + } + } + return count; + } + + /** + * What each stored request carries, in queue order: the key of its first event, or + * {@code "crash"} for a crash request. + * + * @return one entry per request + */ + private static List contentOfEachRequest() { + List contents = new ArrayList<>(); + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey("crash")) { + contents.add("crash"); + } else { + contents.add(new JSONArray(request.get("events")).getJSONObject(0).getString("key")); + } + } + return contents; + } + + /** + * The crash objects of the stored crash requests, in queue order. + * + * @return the crash objects + */ + private static List crashRequests() { + List crashes = new ArrayList<>(); + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey("crash")) { + crashes.add(new JSONObject(request.get("crash"))); + } + } + return crashes; + } + + /** + * The {@code [CLY]_view} events in the event queue, in queue order. + * + * @return the view events + */ + private static List viewEvents() { + List views = new ArrayList<>(); + for (EventImpl event : TestUtils.getCurrentEQ()) { + if (ModuleViews.KEY_VIEW_EVENT.equals(event.key)) { + views.add(event); + } + } + return views; + } + + /** + * How many files in the storage directory start with a prefix. + * + * @param prefix the file name prefix + * @return the number of files + */ + private static int filesWithPrefix(String prefix) { + File[] files = TestUtils.getTestSDirectory().listFiles((dir, name) -> name.startsWith(prefix)); + return files == null ? 0 : files.length; + } + + /** + * Asserts that a request carries no location parameter, not even the empty one that erases it. + * + * @param request the request parameters + */ + private static void assertNoLocation(Map request) { + for (String param : Arrays.asList("location", "city", "country_code", "ip")) { + Assert.assertFalse("carries " + param + ": " + request, request.containsKey(param)); + } + } + + /** + * Adds numbered breadcrumbs, {@code prefix1} to {@code prefixN}. + * + * @param prefix the text before the number + * @param count how many + */ + private static void addBreadcrumbs(String prefix, int count) { + for (int i = 1; i <= count; i++) { + Countly.instance().crashes().addCrashBreadcrumb(prefix + i); + } + } + + /** + * Throws on a new thread and waits for it to end, which is after the default uncaught exception + * handler returned. + * + * @param crash what the thread throws + */ + private static void crashOnAThread(RuntimeException crash) throws InterruptedException { + Thread thread = new Thread(() -> { + throw crash; + }); + thread.start(); + thread.join(); + } + + /** + * The settings test configuration with a log listener that keeps the probe lines of the logging test. + * + * @param heard where the probe lines go + * @return the configuration + */ + private static Config configWithProbeListener(List heard) { + return TestUtils.getConfigSdkBehaviorSettings().setLogListener((message, level) -> { + if (message.startsWith("probe_")) { + heard.add(message); + } + }); + } + + /** + * What was printed so far. + * + * @param printed the captured output + * @return the text + */ + private static String printedText(ByteArrayOutputStream printed) { + return new String(printed.toByteArray(), StandardCharsets.UTF_8); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationLimitsTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationLimitsTests.java new file mode 100644 index 00000000..e718ee49 --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationLimitsTests.java @@ -0,0 +1,754 @@ +package ly.count.sdk.java.internal; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import ly.count.sdk.java.Crash; +import ly.count.sdk.java.CrashProcessor; +import ly.count.sdk.java.PredefinedUserPropertyKeys; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * The SDK internal limits of the SDK behavior settings, {@code lkl}, {@code lvs}, {@code lsv}, + * {@code ltlpt} and {@code ltl}, with the breadcrumb limit {@code lbc}, observed through the public + * API: what custom events, timed events, views, user properties, crashes and feedback widget results + * send once the settings set limits, how a later response changes them, and that nothing is cut + * while the settings set none. + *

+ * Settings requests go through a {@link ModuleConfigurationTests.ServerConfigResponder}, which answers + * on the calling thread, so the response of the fetch made at init applies before init returns and a + * later one is pushed by fetching the way the refresh timer does. Networking is off through the + * settings wherever they are fetched, so every stored request stays on disk to be read. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationLimitsTests { + + private ModuleConfigurationTests.ServerConfigResponder server; + private final List warnings = new CopyOnWriteArrayList<>(); + + /** + * Starts every test from empty storage, without a test module, with a fresh responder and with + * the crash processor passing crashes through. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new ModuleConfigurationTests.ServerConfigResponder(); + warnings.clear(); + DumpingCrashProcessor.replaceWithDump = false; + DumpingCrashProcessor.framesPerThread = 5; + } + + /** + * Stops the SDK and clears its data. + */ + @After + public void afterTest() { + Countly.instance().halt(); + DumpingCrashProcessor.replaceWithDump = false; + } + + // region scenarios + + /** + * {@code lkl}, {@code lvs} and {@code lsv} cut custom events after the listing filters and the + * journey trigger match, which see the full key, while an internal event only gets the string + * values of its widget result cut. + *

+ * Verifies on the wire that a journey trigger listed by its full key sends the queue at once, that + * its key is cut to 5 characters, that a segmentation key blacklisted by its full name and an + * entry of an unsupported type are removed before the entry limit counts while a key blacklisted + * only by its truncated form is kept, that of two keys cut to the same key the one that sorts last + * keeps its value, that string values are cut to 6 characters, that past 2 entries the keys that + * sort last are dropped and that the caller's map is left as it was; that an event whose truncated + * key is whitelisted but whose full + * key is not is dropped; that a timed event is cut the same way; and that a feedback widget + * result keeps every key and entry and the values the SDK adds, only its own strings being cut. + */ + @Test + public void customEvents_areCutAfterTheFiltersAndTheJourneyTriggerSawTheFullKey_andInternalEventsOnlyGetWidgetValuesCut() { + server.respondWith(smallLimits() + .eventFilterList(names("purchase_completed", "timed_checkout", "check"), true) + .segmentationFilterList(names("internal_notes", "payme"), false) + .journeyTriggerEvents(names("purchase_completed"))); + init(configWithWarnings().enableFeatures(Config.Feature.Events, Config.Feature.Feedback).setEventQueueSizeToSend(100)); + + Map segmentation = new HashMap<>(); + segmentation.put("payment_method", "credit_card"); + segmentation.put("category_a", "books"); + segmentation.put("category_b", "movies"); + segmentation.put("internal_notes", "removed_by_the_filter"); + segmentation.put("aa_unsupported", new Object()); + segmentation.put("zz_note", "dropped_by_the_entry_limit"); + Map callers = Collections.unmodifiableMap(new HashMap<>(segmentation)); + Countly.instance().events().recordEvent("purchase_completed", callers, 2, 9.99); + TestUtils.letTheClockCatchUp(); + + List sent = sentEvents(); + Assert.assertEquals(keysOf(sent).toString(), 1, sent.size()); + assertEvent(sent.get(0), "purch", TestUtils.map("categ", "movies", "payme", "credit")); + Assert.assertEquals(2, sent.get(0).count); + Assert.assertEquals(9.99, sent.get(0).sum, 0.0); + Assert.assertEquals(segmentation, callers); + Assert.assertEquals(1, warningsContaining("[ModuleEvents] recordEventInternal, the key [purchase_completed] is longer than the key length limit of [5]")); + Assert.assertEquals(1, warningsContaining("its value replaces that of [categ]")); + Assert.assertEquals(1, warningsContaining("the segmentation has [3] entries, over the limit of [2]")); + + Countly.instance().events().recordEvent("checkout_started"); + Assert.assertEquals(1, warningsContaining("is filtered out by the event filter")); + + Countly.instance().events().startEvent("timed_checkout"); + Countly.instance().events().endEvent("timed_checkout", TestUtils.map("duration_bucket", "long_running", "attempt", 2), 1, null); + + CountlyFeedbackWidget widget = new CountlyFeedbackWidget(); + widget.widgetId = "rating_widget_identifier"; + widget.type = FeedbackWidgetType.rating; + widget.name = "rating"; + widget.tags = new String[0]; + Map widgetResult = new HashMap<>(); + widgetResult.put("rating", 4); + widgetResult.put("comment", "a comment longer than six characters"); + widgetResult.put("email", "user@example.com"); + widgetResult.put("contactMe", true); + Countly.instance().feedback().reportFeedbackWidgetManually(widget, null, widgetResult); + + List queued = TestUtils.getCurrentEQ(); + Assert.assertEquals(keysOf(queued).toString(), 2, queued.size()); + assertEvent(queued.get(0), "timed", TestUtils.map("attem", 2, "durat", "long_r")); + assertEvent(queued.get(1), FeedbackWidgetType.rating.eventKey, TestUtils.map("platform", WidgetUrlBuilder.PLATFORM, + "app_version", TestUtils.APPLICATION_VERSION, "widget_id", "rating_widget_identifier", + "rating", 4, "comment", "a comm", "email", "user@e", "contactMe", true)); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + } + + /** + * {@code lkl}, {@code lvs} and {@code lsv} cut the name and the segmentation of views, while the + * keys the SDK adds are neither cut nor counted. + *

+ * Verifies on the wire that the name is cut to 5 characters on the start and the end event, that + * the global, the added and the given segmentation are merged and then cut, that a key cut to a + * reserved key is removed so an end event never carries {@code visit}, that an entry of an + * unsupported type takes no place under the entry limit, that past 2 entries the keys that sort + * last are dropped while {@code name}, {@code visit}, {@code start} and + * {@code segment} stay with their values uncut, that a journey trigger view matches the name as + * sent, and that a view started under a long name is still stopped by that name. + */ + @Test + public void views_nameAndSegmentationAreCut_whileTheKeysTheSdkAddsAreNeitherCutNorCounted() { + server.respondWith(smallLimits().journeyTriggerViews(names("Setti"))); + init(configWithWarnings().enableFeatures(Config.Feature.Events, Config.Feature.Views).setEventQueueSizeToSend(100)); + String platform = SDKCore.instance.config.getSdkPlatform(); + Countly.instance().views().setGlobalViewSegmentation(TestUtils.map("global_key", "global_value")); + + String viewId = Countly.instance().views().startView("SettingsScreen", TestUtils.map("visitor_type", "returning", "aa_first", 1)); + TestUtils.letTheClockCatchUp(); + Countly.instance().views().addSegmentationToViewWithID(viewId, TestUtils.map("added_key", "added_value", "bb_second", 2)); + Countly.instance().views().stopViewWithID(viewId, TestUtils.map("visitor_type", "new_visitor", "cc_third", "third_value", "aa_unsupported", new int[] { 1 })); + TestUtils.letTheClockCatchUp(); + + List sent = sentEvents(); + Assert.assertEquals(keysOf(sent).toString(), 2, sent.size()); + assertEvent(sent.get(0), ModuleViews.KEY_VIEW_EVENT, TestUtils.map("aa_fi", 1, "globa", "global", + ModuleViews.KEY_NAME, "Setti", ModuleViews.KEY_VISIT, "1", ModuleViews.KEY_START, "1", ModuleViews.KEY_SEGMENT, platform)); + assertEvent(sent.get(1), ModuleViews.KEY_VIEW_EVENT, TestUtils.map("added", "added_", "bb_se", 2, + ModuleViews.KEY_NAME, "Setti", ModuleViews.KEY_SEGMENT, platform)); + Assert.assertEquals(viewId, sent.get(1).id); + Assert.assertEquals(2, warningsContaining("[ModuleViews] createViewEventSegmentation, a key truncated to the reserved key [visit] is removed")); + + Countly.instance().views().startView("Home"); + Countly.instance().views().startView("ProfileScreen"); + Countly.instance().views().stopViewWithName("ProfileScreen"); + + List queued = TestUtils.getCurrentEQ(); + Assert.assertEquals(keysOf(queued).toString(), 3, queued.size()); + assertEvent(queued.get(0), ModuleViews.KEY_VIEW_EVENT, TestUtils.map("globa", "global", + ModuleViews.KEY_NAME, "Home", ModuleViews.KEY_VISIT, "1", ModuleViews.KEY_SEGMENT, platform)); + Assert.assertEquals("Profi", queued.get(1).segmentation.get(ModuleViews.KEY_NAME)); + Assert.assertEquals("1", queued.get(1).segmentation.get(ModuleViews.KEY_VISIT)); + assertEvent(queued.get(2), ModuleViews.KEY_VIEW_EVENT, TestUtils.map("globa", "global", + ModuleViews.KEY_NAME, "Profi", ModuleViews.KEY_SEGMENT, platform)); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + } + + /** + * {@code lkl}, {@code lvs} and {@code lsv} cut custom user properties, sets and modifications, + * while predefined properties only get their string values cut and the picture is never cut. The + * limits in effect when the request is built apply. + *

+ * Verifies on the wire that predefined keys stay whole, that the name, email and organization + * are cut to 6 characters while the picture URL, the gender and the birth year are sent as they + * are; that custom keys are cut to 5 characters, the key that sorts last winning a collision, + * string values cut to 6, and past 2 sets the keys that sort last are dropped; that modification + * keys and string operands are cut the same way and never count against the entry limit; and + * that a property set before a response lowered the key length limit is cut by the new limit. + */ + @Test + public void userProperties_customKeysAndValuesAreCutForSetsAndModifications_predefinedOnlyTheirValuesAndNeverThePicture() { + server.respondWith(smallLimits()); + init(configWithWarnings()); + + String pictureUrl = "https://example.com/pictures/avatar.png"; + Map batch = new LinkedHashMap<>(); + batch.put(PredefinedUserPropertyKeys.NAME, "Johnathan Doe"); + batch.put(PredefinedUserPropertyKeys.EMAIL, "johnathan@example.com"); + batch.put(PredefinedUserPropertyKeys.ORGANIZATION, "Countly Ltd"); + batch.put(PredefinedUserPropertyKeys.PICTURE_PATH, pictureUrl); + batch.put(PredefinedUserPropertyKeys.GENDER, "M"); + batch.put(PredefinedUserPropertyKeys.BIRTH_YEAR, 1990); + batch.put("favourite_colour", "turquoise"); + batch.put("favourite_food", "spaghetti"); + batch.put("aaa_first", 1); + batch.put("zzz_last", "dropped"); + Countly.instance().userProfile().setProperties(batch); + Countly.instance().userProfile().increment("login_count_total"); + Countly.instance().userProfile().push("visited_pages", "homepage_long"); + Countly.instance().userProfile().setOnce("first_seen_at", "2024-01-01"); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + JSONObject details = userDetails(0); + Assert.assertEquals("Johnat", details.getString(PredefinedUserPropertyKeys.NAME)); + Assert.assertEquals("johnat", details.getString(PredefinedUserPropertyKeys.EMAIL)); + Assert.assertEquals("Countl", details.getString(PredefinedUserPropertyKeys.ORGANIZATION)); + Assert.assertEquals(pictureUrl, details.getString(PredefinedUserPropertyKeys.PICTURE)); + Assert.assertEquals("M", details.getString(PredefinedUserPropertyKeys.GENDER)); + Assert.assertEquals(1990, details.getInt(PredefinedUserPropertyKeys.BIRTH_YEAR)); + JSONObject custom = details.getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(names("aaa_f", "favou", "login", "visit", "first"), custom.keySet()); + Assert.assertEquals(1, custom.getInt("aaa_f")); + Assert.assertEquals("spaghe", custom.getString("favou")); + Assert.assertEquals(1, custom.getJSONObject("login").getInt("$inc")); + Assert.assertEquals("homepa", custom.getJSONObject("visit").getString("$push")); + Assert.assertEquals("2024-0", custom.getJSONObject("first").getString("$setOnce")); + Assert.assertEquals(1, warningsContaining("[ModuleUserProfile] perform, the segmentation has [3] entries, over the limit of [2]")); + + Countly.instance().userProfile().setProperty("abc_property", "value"); + Countly.instance().userProfile().incrementBy("xyz_counter", 3); + push(new ServerConfigBuilder().keyLengthLimit(3)); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + custom = userDetails(1).getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(names("abc", "xyz"), custom.keySet()); + Assert.assertEquals("value", custom.getString("abc")); + Assert.assertEquals(3, custom.getJSONObject("xyz").getInt("$inc")); + } + + /** + * {@code ltl} cuts every stack trace line, {@code ltlpt} the lines of each thread of a trace dump, + * {@code lkl}, {@code lvs} and {@code lsv} the custom segments, and {@code lvs} and {@code lbc} + * the breadcrumbs, including what a crash processor adds. + *

+ * Verifies on the wire that a handled exception keeps every line of its stack trace, each cut to + * 20 characters, that its segments are cut to 5 character keys and 6 character values with the 2 + * keys that sort first kept, and that only the 2 newest breadcrumbs are sent, cut to 6 characters; + * and that when a crash processor replaces the stack trace with a dump of two threads and sets its + * own segments and breadcrumbs, each thread keeps its 3 top lines, the lines are cut to 20 + * characters and the segments and breadcrumbs it set are cut as well. + */ + @Test + public void crashes_stackTraceLinesSegmentsAndBreadcrumbsAreCut_includingWhatACrashProcessorAdds() { + server.respondWith(smallLimits().breadcrumbLimit(2)); + init(configWithWarnings().enableFeatures(Config.Feature.CrashReporting).setCrashProcessorClass(DumpingCrashProcessor.class)); + + Countly.instance().crashes().addCrashBreadcrumb("first_breadcrumb"); + Countly.instance().crashes().addCrashBreadcrumb("second_breadcrumb"); + Countly.instance().crashes().addCrashBreadcrumb("third_breadcrumb"); + Exception exception = new Exception("a message longer than twenty characters"); + Countly.instance().crashes().recordHandledException(exception, TestUtils.map("screen_name", "checkout_page", "retry_count", 3, "zz_last", "value")); + TestUtils.letTheClockCatchUp(); + + DumpingCrashProcessor.replaceWithDump = true; + Countly.instance().crashes().recordUnhandledException(new RuntimeException("dumped")); + TestUtils.letTheClockCatchUp(); + + List crashes = crashRequests(); + Assert.assertEquals(2, crashes.size()); + + JSONObject handled = crashes.get(0); + String fullStackTrace = stackTraceOf(exception); + Assert.assertTrue(lineCount(fullStackTrace) > 3); + Assert.assertEquals(truncateLines(fullStackTrace, 20), handled.getString("_error")); + Assert.assertEquals(lineCount(fullStackTrace), lineCount(handled.getString("_error"))); + Assert.assertTrue(handled.getBoolean("_nonfatal")); + Assert.assertEquals("second\nthird_", handled.getString("_logs")); + JSONObject segments = handled.getJSONObject("_custom"); + Assert.assertEquals(names("retry", "scree"), segments.keySet()); + Assert.assertEquals(3, segments.getInt("retry")); + Assert.assertEquals("checko", segments.getString("scree")); + + JSONObject dumped = crashes.get(1); + String frameLine = "\tat ly.count.Example\n"; + String expectedDump = "Thread [main]:" + System.lineSeparator() + frameLine + frameLine + frameLine + "\n\n" + + "Thread [worker-threa\n" + frameLine + frameLine + frameLine + "\n\n"; + Assert.assertEquals("anr", dumped.getString("_type")); + Assert.assertFalse(dumped.getBoolean("_nonfatal")); + Assert.assertEquals(expectedDump, dumped.getString("_error")); + segments = dumped.getJSONObject("_custom"); + Assert.assertEquals(names("anoth", "proce"), segments.keySet()); + Assert.assertEquals("anothe", segments.getString("anoth")); + Assert.assertEquals("proces", segments.getString("proce")); + Assert.assertEquals("proces", dumped.getString("_logs")); + Assert.assertEquals(2, warningsContaining("[ModuleCrash] onCrash, the stack trace of the thread")); + } + + /** + * A response that changes a limit applies from the next call, lowering or raising it. + *

+ * Verifies that an event recorded before any limit is sent whole, that after a response brings + * key and value limits the same call is cut by them, that a later response lowering the key + * length and adding an entry limit lifts the value limit it leaves out, that one raising + * every limit sends the event whole again from the caller's untouched map, and that breadcrumbs + * and stack trace lines follow the limits in effect when each breadcrumb is added and when the + * crash is recorded. + */ + @Test + public void aResponseChangingTheLimits_appliesFromTheNextCall() { + server.respondWith(new ServerConfigBuilder().networking(false)); + init(configWithWarnings().enableFeatures(Config.Feature.Events, Config.Feature.CrashReporting).setEventQueueSizeToSend(100)); + Map segmentation = TestUtils.map("segment_key", "segment_value", "second", 2); + + Countly.instance().events().recordEvent("event_key", segmentation); + push(new ServerConfigBuilder().keyLengthLimit(8).valueSizeLimit(4)); + Countly.instance().events().recordEvent("event_key", segmentation); + push(new ServerConfigBuilder().keyLengthLimit(3).segmentationValuesLimit(1)); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider().getMaxValueSize()); + Countly.instance().events().recordEvent("event_key", segmentation); + push(new ServerConfigBuilder().keyLengthLimit(64).valueSizeLimit(64).segmentationValuesLimit(64)); + Countly.instance().events().recordEvent("event_key", segmentation); + + List queued = TestUtils.getCurrentEQ(); + Assert.assertEquals(keysOf(queued).toString(), 4, queued.size()); + assertEvent(queued.get(0), "event_key", segmentation); + assertEvent(queued.get(1), "event_ke", TestUtils.map("segment_", "segm", "second", 2)); + assertEvent(queued.get(2), "eve", TestUtils.map("sec", 2)); + assertEvent(queued.get(3), "event_key", segmentation); + + push(new ServerConfigBuilder().valueSizeLimit(4)); + Countly.instance().crashes().addCrashBreadcrumb("alpha_breadcrumb"); + push(new ServerConfigBuilder().valueSizeLimit(4).breadcrumbLimit(1).traceLengthLimit(12)); + Countly.instance().crashes().addCrashBreadcrumb("omega_breadcrumb"); + Exception exception = new Exception("recorded after the change"); + Countly.instance().crashes().recordHandledException(exception); + TestUtils.letTheClockCatchUp(); + + List crashes = crashRequests(); + Assert.assertEquals(1, crashes.size()); + Assert.assertEquals("omeg", crashes.get(0).getString("_logs")); + Assert.assertEquals(truncateLines(stackTraceOf(exception), 12), crashes.get(0).getString("_error")); + } + + /** + * With a live response that sets no limit, nothing any feature sends is cut, however long or wide. + *

+ * Verifies that the provider reports no key, value, entry, lines per thread and line length limit + * and the developer breadcrumb limit, and then the same as + * {@link #assertNothingIsCut()}. + */ + @Test + public void withSettingsThatSetNoLimit_nothingIsCut() { + server.respondWith(new ServerConfigBuilder().networking(false)); + init(configWithWarnings().enableFeatures(Config.Feature.Events, Config.Feature.Views, Config.Feature.CrashReporting, Config.Feature.Feedback) + .setEventQueueSizeToSend(1000).setCrashProcessorClass(DumpingCrashProcessor.class)); + + ConfigurationProvider provider = provider(); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider.getMaxKeyLength()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider.getMaxValueSize()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider.getMaxSegmentationValues()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider.getMaxStackTraceLinesPerThread()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider.getMaxStackTraceLineLength()); + Assert.assertEquals(100, provider.getMaxBreadcrumbCount()); + + assertNothingIsCut(); + } + + /** + * Without any SDK behavior settings, the settings requests being disabled as for an upgrading app + * that never received any, nothing any feature sends is cut, however long or wide. + *

+ * Verifies the same as {@link #assertNothingIsCut()}. + */ + @Test + public void withoutSettings_nothingIsCut() { + Countly.instance().init(TestUtils.getBaseConfig().setLogListener(this::keepWarning) + .enableFeatures(Config.Feature.Events, Config.Feature.Views, Config.Feature.CrashReporting, Config.Feature.Feedback) + .setEventQueueSizeToSend(1000).setCrashProcessorClass(DumpingCrashProcessor.class)); + + assertNothingIsCut(); + } + + // endregion + // region helpers + + /** + * Records a 200 character event key with 150 segments of 200 character keys and 300 character + * values as an event, a timed event and a view with its end event, reports a feedback widget + * result with a 300 character comment, sets 150 such custom user properties with a 300 character + * name and a modification, and records a crash with that segmentation, a 300 character message + * and breadcrumb and a crash processor dump of 40 lines per thread. Asserts that every key, value, + * entry and stack trace line is sent whole and that no limit warning was logged. + */ + private void assertNothingIsCut() { + String longKey = repeat("key_", 50); + String longValue = repeat("value_", 50); + Map wide = new HashMap<>(); + for (int i = 0; i < 150; i++) { + wide.put(longKey + i, longValue + i); + } + + Countly.instance().events().recordEvent(longKey, wide); + Countly.instance().events().startEvent(longKey + "timed"); + Countly.instance().events().endEvent(longKey + "timed", wide, 1, null); + String viewId = Countly.instance().views().startView(longKey, wide); + Countly.instance().views().stopViewWithID(viewId, wide); + CountlyFeedbackWidget widget = new CountlyFeedbackWidget(); + widget.widgetId = "rating_widget_identifier"; + widget.type = FeedbackWidgetType.rating; + widget.name = "rating"; + widget.tags = new String[0]; + Countly.instance().feedback().reportFeedbackWidgetManually(widget, null, TestUtils.map("rating", 5, "comment", longValue)); + + List queued = TestUtils.getCurrentEQ(); + Assert.assertEquals(keysOf(queued).toString(), 5, queued.size()); + assertEvent(queued.get(0), longKey, wide); + assertEvent(queued.get(1), longKey + "timed", wide); + Map viewStart = new HashMap<>(queued.get(2).segmentation); + Assert.assertEquals(longKey, viewStart.remove(ModuleViews.KEY_NAME)); + Assert.assertEquals("1", viewStart.remove(ModuleViews.KEY_VISIT)); + Assert.assertEquals("1", viewStart.remove(ModuleViews.KEY_START)); + Assert.assertNotNull(viewStart.remove(ModuleViews.KEY_SEGMENT)); + Assert.assertEquals(wide, viewStart); + Map viewEnd = new HashMap<>(queued.get(3).segmentation); + Assert.assertEquals(longKey, viewEnd.remove(ModuleViews.KEY_NAME)); + Assert.assertNotNull(viewEnd.remove(ModuleViews.KEY_SEGMENT)); + Assert.assertEquals(wide, viewEnd); + Assert.assertEquals(longValue, queued.get(4).segmentation.get("comment")); + + Map properties = new HashMap<>(wide); + properties.put(PredefinedUserPropertyKeys.NAME, longValue); + Countly.instance().userProfile().setProperties(properties); + Countly.instance().userProfile().push(longKey + "list", longValue); + Countly.instance().userProfile().save(); + TestUtils.letTheClockCatchUp(); + + JSONObject details = null; + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey("user_details")) { + details = new JSONObject(request.get("user_details")); + } + } + Assert.assertNotNull(details); + Assert.assertEquals(longValue, details.getString(PredefinedUserPropertyKeys.NAME)); + JSONObject custom = details.getJSONObject(ModuleUserProfile.CUSTOM_KEY); + Assert.assertEquals(151, custom.length()); + for (Map.Entry entry : wide.entrySet()) { + Assert.assertEquals(entry.getValue(), custom.getString(entry.getKey())); + } + Assert.assertEquals(longValue, custom.getJSONObject(longKey + "list").getString("$push")); + + Countly.instance().crashes().addCrashBreadcrumb(longValue); + Exception exception = new Exception(longValue); + Countly.instance().crashes().recordHandledException(exception, wide); + TestUtils.letTheClockCatchUp(); + DumpingCrashProcessor.replaceWithDump = true; + DumpingCrashProcessor.framesPerThread = 40; + Countly.instance().crashes().recordHandledException(new Exception("dumped")); + TestUtils.letTheClockCatchUp(); + + List crashes = crashRequests(); + Assert.assertEquals(2, crashes.size()); + Assert.assertEquals(stackTraceOf(exception), crashes.get(0).getString("_error")); + Assert.assertEquals(longValue, crashes.get(0).getString("_logs")); + JSONObject segments = crashes.get(0).getJSONObject("_custom"); + Assert.assertEquals(wide.size(), segments.length()); + for (Map.Entry entry : wide.entrySet()) { + Assert.assertEquals(entry.getValue(), segments.getString(entry.getKey())); + } + String dump = crashes.get(1).getString("_error"); + Assert.assertEquals(80, dump.split("\tat ly.count.Example.method", -1).length - 1); + Assert.assertTrue(dump.contains("Thread [worker-thread]:\n")); + Assert.assertEquals(3, crashes.get(1).getJSONObject("_custom").length()); + Assert.assertEquals("processor_breadcrumb", crashes.get(1).getString("_logs")); + + Assert.assertEquals(0, warningsContaining("limit of [")); + Assert.assertEquals(0, warningsContaining("reserved key")); + } + + /** + * The limits most scenarios run with: keys of 5 characters, values of 6, 2 segmentation entries + * and 3 stack trace lines per thread of 20 characters each, with networking off. + * + * @return the settings + */ + private static ServerConfigBuilder smallLimits() { + return new ServerConfigBuilder().keyLengthLimit(5).valueSizeLimit(6).segmentationValuesLimit(2) + .traceLinesLimit(3).traceLengthLimit(20).networking(false); + } + + /** + * Initializes the SDK with {@link #server} answering its settings requests. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + server.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * The settings test configuration with a listener that keeps every warning. + * + * @return the configuration + */ + private Config configWithWarnings() { + return TestUtils.getConfigSdkBehaviorSettings().setLogListener(this::keepWarning); + } + + /** + * A log listener that keeps every warning. + * + * @param message the log line + * @param level its level + */ + private void keepWarning(String message, Config.LoggingLevel level) { + if (level == Config.LoggingLevel.WARN) { + warnings.add(message); + } + } + + /** + * How many warnings so far contain a text. + * + * @param text the text + * @return the number of warnings + */ + private int warningsContaining(String text) { + int count = 0; + for (String warning : warnings) { + if (warning.contains(text)) { + count++; + } + } + return count; + } + + /** + * Every event of every stored request, in queue order. + * + * @return the events + */ + private static List sentEvents() { + List events = new ArrayList<>(); + int requestCount = TestUtils.getCurrentRQ().length; + for (int i = 0; i < requestCount; i++) { + events.addAll(TestUtils.readEventsFromRequest(i, TestUtils.DEVICE_ID)); + } + return events; + } + + /** + * The keys of events, in order. + * + * @param events the events + * @return the keys + */ + private static List keysOf(List events) { + List keys = new ArrayList<>(); + for (EventImpl event : events) { + keys.add(event.key); + } + return keys; + } + + /** + * Asserts the key and the whole segmentation of an event. + * + * @param event the event + * @param key the expected key + * @param segmentation the expected segmentation + */ + private static void assertEvent(EventImpl event, String key, Map segmentation) { + Assert.assertEquals(key, event.key); + Assert.assertEquals(segmentation, new HashMap<>(event.segmentation)); + } + + /** + * The {@code user_details} object of a stored request. + * + * @param requestIndex the index of the request in the queue + * @return the user details + */ + private static JSONObject userDetails(int requestIndex) { + Map[] requests = TestUtils.getCurrentRQ(); + Assert.assertTrue("no request at index " + requestIndex + " in " + Arrays.toString(requests), requests.length > requestIndex); + return new JSONObject(requests[requestIndex].get("user_details")); + } + + /** + * The crash of every stored crash request, in queue order. + * + * @return the crashes + */ + private static List crashRequests() { + List crashes = new ArrayList<>(); + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey("crash")) { + crashes.add(new JSONObject(request.get("crash"))); + } + } + return crashes; + } + + /** + * The stack trace of a throwable as the SDK prints it. + * + * @param throwable the throwable + * @return the stack trace + */ + private static String stackTraceOf(Throwable throwable) { + StringWriter writer = new StringWriter(); + throwable.printStackTrace(new PrintWriter(writer)); + return writer.toString(); + } + + /** + * Cuts every line of a text to a length, keeping its line breaks. + * + * @param text the text + * @param maxLength the length + * @return the cut text + */ + private static String truncateLines(String text, int maxLength) { + StringBuilder truncated = new StringBuilder(); + String[] lines = text.split("\n", -1); + for (int i = 0; i < lines.length; i++) { + String line = lines[i]; + String carriageReturn = ""; + if (line.endsWith("\r")) { + line = line.substring(0, line.length() - 1); + carriageReturn = "\r"; + } + truncated.append(line.length() > maxLength ? line.substring(0, maxLength) : line).append(carriageReturn); + if (i < lines.length - 1) { + truncated.append('\n'); + } + } + return truncated.toString(); + } + + /** + * The number of line breaks of a text. + * + * @param text the text + * @return the number of lines that end with a line break + */ + private static int lineCount(String text) { + return text.split("\n", -1).length - 1; + } + + /** + * A text repeated. + * + * @param text the text + * @param times how many times + * @return the repeated text + */ + private static String repeat(String text, int times) { + StringBuilder repeated = new StringBuilder(text.length() * times); + for (int i = 0; i < times; i++) { + repeated.append(text); + } + return repeated.toString(); + } + + /** + * A crash processor that, when asked to, replaces the stack trace with a dump of a main and a + * worker thread and sets its own segments and breadcrumbs, as a customer's processor may. + */ + public static class DumpingCrashProcessor implements CrashProcessor { + static volatile boolean replaceWithDump = false; + static volatile int framesPerThread = 5; + private static final Thread MAIN_THREAD = new Thread(() -> { + }, "main"); + private static final Thread WORKER_THREAD = new Thread(() -> { + }, "worker-thread"); + + /** + * Replaces the stack trace, the segments and the breadcrumbs when asked to. + * + * @param crash the crash + * @return the same crash, never vetoed + */ + @Override + public Crash process(Crash crash) { + if (replaceWithDump) { + Map traces = new LinkedHashMap<>(); + traces.put(MAIN_THREAD, frames(framesPerThread)); + traces.put(WORKER_THREAD, frames(framesPerThread)); + crash.addTraces(MAIN_THREAD, traces); + Map segments = new HashMap<>(); + segments.put("processor_segment", "processor_value"); + segments.put("another_segment", "another_value"); + segments.put("zzz", "z"); + crash.setSegments(segments); + crash.setLogs(new String[] { "processor_breadcrumb" }); + } + return crash; + } + + /** + * Stack trace lines of a made up class. + * + * @param count how many + * @return the lines + */ + private static StackTraceElement[] frames(int count) { + StackTraceElement[] frames = new StackTraceElement[count]; + for (int i = 0; i < count; i++) { + frames[i] = new StackTraceElement("ly.count.Example", "method" + i, "Example.java", i + 1); + } + return frames; + } + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationNetworkTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationNetworkTests.java new file mode 100644 index 00000000..2ae7b0eb --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationNetworkTests.java @@ -0,0 +1,1018 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.BooleanSupplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * The network resilience settings of the SDK behavior settings, observed through the public API: the + * backoff mechanism, {@code bom}, {@code bom_at}, {@code bom_rqp}, {@code bom_ra} and {@code bom_d}, + * and the request drop age, {@code dort}. + *

+ * Settings requests go through a {@link ModuleConfigurationTests.ServerConfigResponder}, which answers + * on the calling thread. Queued requests go through the real networking stack to a local HTTP server + * that can hold each answer back and records when every request arrived and when its answer started, + * so the gap a backoff leaves between two requests can be measured. Backoff settings are set to + * seconds, never left at their defaults, and no wait relies on the 60 second SDK timer. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationNetworkTests { + + private static final long SLOW_ANSWER_MS = 1_200; + //longer than the wait budget, so a backoff that should not happen fails the wait instead of slowing it + private static final int BACKOFF_LONGER_THAN_THE_BUDGET_SECONDS = 60; + private static final long WAIT_BUDGET_MS = 30_000; + private static final long POLL_MS = 50; + private static final long TIMER_SLACK_MS = 50; + private static final long LATE_RESUME_MS = 5_000; + private static final long MINUTE_MS = 60L * 1000L; + private static final long HOUR_MS = 60L * MINUTE_MS; + + private static final String BACKOFF_STARTED = "holding the request queue back for ["; + private static final String BACKOFF_ENDED = "the backoff is over"; + private static final String NOT_BACKING_OFF = "not backing off"; + private static final String DROPPED_UNSENT = "dropping it without sending"; + private static final String CONTENT_ENDPOINT = "/o/sdk/content?"; + private static final String NO_CONTENT_RESPONSE = "{\"jsonArray\":[{\"result\":\"No content block found!\"}]}"; + + private SettingsAndContentServer server; + private QueueServer queueServer; + private final List logs = new CopyOnWriteArrayList<>(); + + /** + * Starts every test from empty storage, without a test module and with fresh fakes. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new SettingsAndContentServer(); + queueServer = null; + logs.clear(); + } + + /** + * Stops the SDK, clearing its data, and the local HTTP server when one was started. + */ + @After + public void afterTest() { + Countly.instance().halt(); + if (queueServer != null) { + queueServer.stop(); + } + } + + // region scenarios + + /** + * A request the server answers slowly while few requests are queued holds the queue back for + * {@code bom_d}, then the queue drains on its own. + *

+ * Verifies against a local server that after an answer slower than {@code bom_at}, with two + * requests left, the next request arrives only once {@code bom_d} has passed since the server + * started answering; that meanwhile a newly recorded request and nudges of the send loop send + * nothing and the queue stays on disk as it was; that the backoff ends without any nudge; and + * that the fast answers after it hold nothing back. + */ + @Test + public void backoff_slowAnswerWithFewRequestsQueued_holdsTheQueueBackForTheDuration_thenItDrainsOnItsOwn() throws Exception { + startQueueServer(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS); + server.settings.respondWith(new ServerConfigBuilder().backoffAcceptedTimeout(1).backoffDuration(3)); + init(networkedConfig()); + + recordEvents("first", "second", "third"); + awaitLogs(BACKOFF_STARTED, 1); + Assert.assertEquals(1, logsContaining("and [2] requests remain queued, " + BACKOFF_STARTED + "3] seconds")); + Assert.assertEquals(1, queueServer.received.size()); + + Countly.instance().events().recordEvent("during_backoff"); + TestUtils.letTheClockCatchUp(); + Map[] held = TestUtils.getCurrentRQ(); + Assert.assertEquals(3, held.length); + Assert.assertEquals("[\"second\"]", eventKeys(held[0]).toString()); + Assert.assertEquals("[\"third\"]", eventKeys(held[1]).toString()); + Assert.assertEquals("[\"during_backoff\"]", eventKeys(held[2]).toString()); + + nudgeUntil(queueServer.received.get(0).answeredAtNs + 2_000L * 1_000_000L); + Assert.assertEquals(1, queueServer.received.size()); + Assert.assertEquals(3, queuedRequestCount()); + Assert.assertEquals(0, logsContaining(BACKOFF_ENDED)); + + Assert.assertTrue("the queue never drained after the backoff, logs: " + logs, waitFor(() -> queueServer.received.size() == 4 && queuedRequestCount() == 0)); + long gapMs = gapBeforeRequestMs(1); + Assert.assertTrue("the next request came [" + gapMs + "] ms after the slow answer", gapMs >= 3_000 - TIMER_SLACK_MS); + Assert.assertTrue("the queue resumed [" + gapMs + "] ms after the slow answer", gapMs < 3_000 + LATE_RESUME_MS); + Assert.assertEquals(Arrays.asList("first", "second", "third", "during_backoff"), receivedEventKeys()); + Assert.assertEquals(1, logsContaining(BACKOFF_STARTED)); + Assert.assertEquals(1, logsContaining(BACKOFF_ENDED)); + Assert.assertEquals(0, logsContaining(NOT_BACKING_OFF)); + } + + /** + * The backoff needs a slow answer and the requests still queued within {@code bom_rqp} of the + * request queue size. + *

+ * Verifies against a local server, with {@code rqs} 4 and {@code bom_rqp} 0.25, so at most one + * request left, that a slow answer with two requests left holds nothing back, that the next slow + * answer, with one request left, holds the queue back for {@code bom_d}, and that the fast answer + * to the last request holds nothing back. + */ + @Test + public void backoff_needsTheQueueWithinItsShareOfTheQueueSize_andASlowAnswer() throws Exception { + startQueueServer(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS, SLOW_ANSWER_MS); + server.settings.respondWith(new ServerConfigBuilder().requestQueueSize(4).backoffRequestQueuePercentage(0.25) + .backoffAcceptedTimeout(1).backoffDuration(2)); + init(networkedConfig()); + + recordEvents("a", "b", "c"); + Assert.assertTrue("the queue never drained, logs: " + logs, waitFor(() -> queueServer.received.size() == 3 && queuedRequestCount() == 0)); + + Assert.assertEquals(Arrays.asList("a", "b", "c"), receivedEventKeys()); + Assert.assertEquals(1, logsContaining("but [2] requests remain queued, more than [1.0], " + NOT_BACKING_OFF)); + Assert.assertEquals(1, logsContaining("and [1] requests remain queued, " + BACKOFF_STARTED + "2] seconds")); + Assert.assertEquals(1, logsContaining(BACKOFF_STARTED)); + Assert.assertEquals(1, logsContaining(NOT_BACKING_OFF)); + long gapAfterFirstMs = gapBeforeRequestMs(1); + long gapAfterSecondMs = gapBeforeRequestMs(2); + Assert.assertTrue("the second request waited [" + gapAfterFirstMs + "] ms", gapAfterFirstMs < 2_000); + Assert.assertTrue("the third request waited only [" + gapAfterSecondMs + "] ms", gapAfterSecondMs >= 2_000 - TIMER_SLACK_MS); + } + + /** + * The backoff never holds the queue back after a request older than {@code bom_ra}, nor while the + * server turns it off with {@code bom}. + *

+ * Verifies against a local server, with a {@code bom_d} longer than the wait budget so that a + * backoff fails the wait, that two stored requests recorded two hours ago are both sent although + * each is answered slowly, each slow answer logged as coming too late to back off; and, after a + * restart with {@code bom} false from the server, that two new requests answered slowly are sent + * one after the other without any backoff. + */ + @Test + public void backoff_neverAfterARequestOlderThanTheRequestAge_norWhileTheServerTurnsItOff() throws Exception { + startQueueServer(); + long twoHoursAgo = System.currentTimeMillis() - 2 * HOUR_MS; + storeRequest(twoHoursAgo, "old_1", twoHoursAgo); + storeRequest(twoHoursAgo + 1, "old_2", twoHoursAgo + 1); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS, SLOW_ANSWER_MS); + server.settings.respondWith(new ServerConfigBuilder().backoffAcceptedTimeout(1).backoffRequestAge(1) + .backoffDuration(BACKOFF_LONGER_THAN_THE_BUDGET_SECONDS)); + init(networkedConfig()); + + Assert.assertTrue("the old requests never drained, logs: " + logs, waitFor(() -> queueServer.received.size() == 2 && queuedRequestCount() == 0)); + Assert.assertEquals(Arrays.asList("old_1", "old_2"), receivedParams("marker")); + awaitLogs("but the request is older than [1] hours, " + NOT_BACKING_OFF, 2); + Assert.assertEquals(0, logsContaining(BACKOFF_STARTED)); + Countly.instance().halt(); + + queueServer.received.clear(); + logs.clear(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS, SLOW_ANSWER_MS); + server.settings.respondWith(new ServerConfigBuilder().backoffMechanism(false).backoffAcceptedTimeout(1) + .backoffDuration(BACKOFF_LONGER_THAN_THE_BUDGET_SECONDS)); + init(networkedConfig()); + Assert.assertFalse(provider().getBOMEnabled()); + + recordEvents("x", "y"); + Assert.assertTrue("the new requests never drained, logs: " + logs, waitFor(() -> queueServer.received.size() == 2 && queuedRequestCount() == 0)); + Assert.assertEquals(Arrays.asList("x", "y"), receivedEventKeys()); + Assert.assertEquals(0, logsContaining(BACKOFF_STARTED)); + Assert.assertEquals(0, logsContaining(NOT_BACKING_OFF)); + } + + /** + * {@link Config#disableBackoffMechanism()} keeps the backoff off while the settings server stays + * silent, and a {@code bom} true from the server turns it back on. + *

+ * Verifies against a local server, with the settings server never answering and the provided + * settings setting {@code bom_at} and a {@code bom_d} longer than the wait budget, that slow + * answers hold nothing back; then that once a response sets {@code bom} true with a short + * {@code bom_d}, the next slow answer holds the queue back for that long. + */ + @Test + public void backoff_offByTheDeveloper_staysOffWithASilentServer_andTheServerTurnsItBackOn() throws Exception { + startQueueServer(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS, SLOW_ANSWER_MS, SLOW_ANSWER_MS); + server.settings.pending(); + init(networkedConfig().disableBackoffMechanism().setSdkBehaviorSettings(new ServerConfigBuilder() + .backoffAcceptedTimeout(1).backoffDuration(BACKOFF_LONGER_THAN_THE_BUDGET_SECONDS).build())); + Assert.assertFalse(provider().getBOMEnabled()); + Assert.assertEquals(1, provider().getBOMAcceptedTimeoutSeconds()); + + recordEvents("a", "b"); + Assert.assertTrue("the queue never drained, logs: " + logs, waitFor(() -> queueServer.received.size() == 2 && queuedRequestCount() == 0)); + Assert.assertEquals(0, logsContaining(BACKOFF_STARTED)); + + push(new ServerConfigBuilder().backoffMechanism(true).backoffAcceptedTimeout(1).backoffDuration(2)); + Assert.assertTrue(provider().getBOMEnabled()); + Assert.assertEquals(2, provider().getBOMDuration()); + + recordEvents("c", "d"); + Assert.assertTrue("the queue never drained after the backoff, logs: " + logs, waitFor(() -> queueServer.received.size() == 4 && queuedRequestCount() == 0)); + Assert.assertEquals(Arrays.asList("a", "b", "c", "d"), receivedEventKeys()); + Assert.assertEquals(1, logsContaining("and [1] requests remain queued, " + BACKOFF_STARTED + "2] seconds")); + long gapMs = gapBeforeRequestMs(3); + Assert.assertTrue("the last request waited only [" + gapMs + "] ms", gapMs >= 2_000 - TIMER_SLACK_MS); + } + + /** + * Backend mode never holds its request queue back and never drops a request for its age, whatever + * the resolved values say. + *

+ * Verifies against a local server, with the settings inert in backend mode and the resolved + * backoff and drop age values forced to ones that would back off and drop, that three requests, + * the first two answered slowly and the first recorded three hours ago, are all sent in order with + * their own timestamps, that nothing backs off or is dropped, and that no settings request is made. + */ + @Test + public void backendMode_neverHoldsTheQueueBack_norDropsARequestForItsAge() throws Exception { + startQueueServer(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS, SLOW_ANSWER_MS); + init(networkedConfig().enableBackendMode()); + ModuleConfiguration module = SDKCore.instance.module(ModuleConfiguration.class); + module.currentVBOMAcceptedTimeoutSeconds = 1; + module.currentVBOMDuration = BACKOFF_LONGER_THAN_THE_BUDGET_SECONDS; + module.currentVDropAgeHours = 1; + Assert.assertTrue(provider().getBOMEnabled()); + + long threeHoursAgo = System.currentTimeMillis() - 3 * HOUR_MS; + Countly.instance().backendM().sessionBegin("device_old", null, null, threeHoursAgo); + Countly.instance().backendM().sessionBegin("device_fresh", null, null, null); + Countly.instance().backendM().sessionBegin("device_last", null, null, null); + + Assert.assertTrue("the memory queue never drained, logs: " + logs, waitFor(() -> queueServer.received.size() == 3 && memoryQueueSize() == 0)); + Assert.assertEquals(Arrays.asList("device_old", "device_fresh", "device_last"), receivedParams("device_id")); + Assert.assertEquals(String.valueOf(threeHoursAgo), queueServer.received.get(0).params.get("timestamp")); + Assert.assertEquals(0, logsContaining(BACKOFF_STARTED)); + Assert.assertEquals(0, logsContaining(NOT_BACKING_OFF)); + Assert.assertEquals(0, logsContaining(DROPPED_UNSENT)); + Assert.assertTrue(server.settings.requests.isEmpty()); + } + + /** + * Stopping the SDK while the queue is held back ends the backoff for good. + *

+ * Verifies against a local server that once the SDK stopped during a backoff, nothing is sent + * after the backoff would have ended, the held request stays on disk, the backoff thread is gone + * and the backoff never reports its end, and that the next init sends the held request. + */ + @Test + public void backoff_stopWhileHeldBack_sendsNothingAfterwards_leavesNoThread_andTheNextInitSendsTheHeldRequest() throws Exception { + startQueueServer(); + queueServer.holdTheNextAnswers(SLOW_ANSWER_MS); + server.settings.respondWith(new ServerConfigBuilder().backoffAcceptedTimeout(1).backoffDuration(2)); + Set threadsBefore = backoffThreads(); + init(networkedConfig()); + + recordEvents("first", "held"); + awaitLogs(BACKOFF_STARTED, 1); + Set threads = backoffThreads(); + threads.removeAll(threadsBefore); + Assert.assertEquals(1, threads.size()); + Assert.assertEquals(1, queuedRequestCount()); + + Countly.instance().stop(); + Assert.assertTrue("the backoff thread outlived stop()", waitFor(() -> noneAlive(threads))); + sleepUntil(queueServer.received.get(0).answeredAtNs + 3_500L * 1_000_000L); + Assert.assertEquals(1, queueServer.received.size()); + Assert.assertEquals(1, queuedRequestCount()); + Assert.assertEquals(0, logsContaining(BACKOFF_ENDED)); + + init(networkedConfig()); + Assert.assertTrue("the held request was never sent, logs: " + logs, waitFor(() -> queueServer.received.size() == 2 && queuedRequestCount() == 0)); + Assert.assertEquals(Arrays.asList("first", "held"), receivedEventKeys()); + Assert.assertEquals(1, logsContaining(BACKOFF_STARTED)); + } + + /** + * {@code dort} drops the stored requests older than it without sending them and sends the others, + * and {@code dort} 0 drops nothing. + *

+ * Verifies against a local server that the age of a request comes from its {@code timestamp} + * parameter, so an old storage ID with a fresh timestamp is sent and a fresh storage ID with an + * old timestamp is dropped, and from its storage ID when it has no timestamp; that each drop is + * logged with the request it dropped; and that with {@code dort} 0 the same old requests are all + * sent in order. + */ + @Test + public void requestDropAge_dropsTheStoredRequestsOlderThanIt_unsent_andZeroDropsNothing() throws Exception { + startQueueServer(); + long now = System.currentTimeMillis(); + long threeHoursAgo = now - 3 * HOUR_MS; + storeRequest(threeHoursAgo, "old_timestamp", threeHoursAgo); + storeRequest(threeHoursAgo + 1, "fresh_timestamp_old_id", now); + storeRequest(threeHoursAgo + 2, "no_timestamp_old_id", null); + storeRequest(now - 2 * MINUTE_MS, "old_timestamp_fresh_id", now - 2 * HOUR_MS); + storeRequest(now - MINUTE_MS, "within_the_drop_age", now - 30 * MINUTE_MS); + server.settings.failing(); + init(networkedConfig().setSdkBehaviorSettings(new ServerConfigBuilder().dropOldRequestTime(1).build())); + Assert.assertEquals(1, provider().getRequestDropAgeHours()); + + Assert.assertTrue("the queue never drained, logs: " + logs, waitNudging(() -> queuedRequestCount() == 0 && queueServer.received.size() == 2)); + Assert.assertEquals(Arrays.asList("fresh_timestamp_old_id", "within_the_drop_age"), receivedParams("marker")); + Assert.assertEquals(3, logsContaining(DROPPED_UNSENT)); + for (long droppedId : new long[] { threeHoursAgo, threeHoursAgo + 2, now - 2 * MINUTE_MS }) { + Assert.assertEquals(1, logsContaining("request [" + droppedId + "] is older than the request drop age of [1] hours")); + } + + TestUtils.createCleanTestState(); + queueServer.received.clear(); + logs.clear(); + storeRequest(threeHoursAgo, "old_timestamp", threeHoursAgo); + storeRequest(threeHoursAgo + 2, "no_timestamp_old_id", null); + storeRequest(now - 2 * MINUTE_MS, "old_timestamp_fresh_id", now - 2 * HOUR_MS); + init(networkedConfig().setSdkBehaviorSettings(new ServerConfigBuilder().dropOldRequestTime(0).build())); + Assert.assertEquals(0, provider().getRequestDropAgeHours()); + + Assert.assertTrue("the queue never drained, logs: " + logs, waitNudging(() -> queuedRequestCount() == 0 && queueServer.received.size() == 3)); + Assert.assertEquals(Arrays.asList("old_timestamp", "no_timestamp_old_id", "old_timestamp_fresh_id"), receivedParams("marker")); + Assert.assertEquals(0, logsContaining(DROPPED_UNSENT)); + } + + /** + * A journey trigger whose request is dropped for its age is settled, while a later trigger still + * refreshes the content zone. + *

+ * Verifies against a local server that a trigger request recorded while networking is off and + * aged past {@code dort} on disk is dropped unsent once networking is back on, that its owner is + * told with {@link Transport#NO_RESPONSE_CODE}, which leaves no pending trigger and the zone + * inactive, and that a fresh trigger is then sent, accepted and enters the zone. + */ + @Test + public void requestDropAge_settlesTheJourneyTriggerOfADroppedRequest_andALaterTriggerStillRefreshes() throws Exception { + startQueueServer(); + server.settings.failing(); + init(networkedConfig().enableFeatures(Config.Feature.Content).setEventQueueSizeToSend(100) + .setSdkBehaviorSettings(new ServerConfigBuilder().networking(false).journeyTriggerEvents(names("purchase")).dropOldRequestTime(1).build())); + holdTheZoneTimer(); + Countly.instance().content().setContentDisplay(new FakeDisplay()); + assertZoneIsInactive(0); + + Countly.instance().events().recordEvent("purchase"); + TestUtils.letTheClockCatchUp(); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + Set pending = SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds; + Assert.assertEquals(storedRequestIds(), pending); + Assert.assertEquals(1, pending.size()); + ageStoredRequests(2 * HOUR_MS); + + push(new ServerConfigBuilder().networking(true).journeyTriggerEvents(names("purchase")).dropOldRequestTime(1)); + Assert.assertTrue("the aged request was never dropped, logs: " + logs, waitNudging(() -> queuedRequestCount() == 0)); + Assert.assertEquals(1, logsContaining(DROPPED_UNSENT)); + awaitLogs("carrying a journey trigger failed with code [" + Transport.NO_RESPONSE_CODE + "], the content zone is not refreshed", 1); + Assert.assertTrue(pending.isEmpty()); + Assert.assertTrue(queueServer.received.isEmpty()); + assertZoneIsInactive(0); + + Countly.instance().events().recordEvent("purchase"); + awaitZoneFetch(1); + Assert.assertEquals(Collections.singletonList("purchase"), receivedEventKeys()); + Assert.assertTrue(pending.isEmpty()); + Assert.assertEquals(1, logsContaining(DROPPED_UNSENT)); + } + + /** + * The request queue trim settles the journey triggers of the requests it drops unread. + *

+ * Verifies on disk, with networking off and {@code rqs} 2, that each journey trigger waits for + * its own request, and that a third trigger, whose request drops the oldest one, leaves waiting + * only the triggers of the two requests still queued. + */ + @Test + public void requestQueueTrim_settlesTheJourneyTriggersOfTheRequestsItDrops() { + server.settings.respondWith(new ServerConfigBuilder().networking(false).requestQueueSize(2).journeyTriggerEvents(names("purchase"))); + init(TestUtils.getConfigSdkBehaviorSettings() + .enableFeatures(Config.Feature.Events) + .setEventQueueSizeToSend(100) + .setLogListener((message, level) -> logs.add(message))); + Set pending = SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds; + + recordEvents("purchase", "purchase"); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + Set firstTwo = storedRequestIds(); + Assert.assertEquals(firstTwo, pending); + long oldest = Collections.min(firstTwo); + + recordEvents("purchase"); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + Set lastTwo = storedRequestIds(); + Assert.assertFalse(lastTwo.contains(oldest)); + Assert.assertEquals(lastTwo, pending); + Assert.assertEquals(1, logsContaining("the request [" + oldest + "] carrying a journey trigger was dropped from the queue without being sent")); + } + + // endregion + // region helpers + + /** + * Stands in for the settings and the content endpoints of the server. Settings requests go to a + * {@link ModuleConfigurationTests.ServerConfigResponder}; content fetches are recorded and + * answered on the calling thread with no content, unless networking is off. + */ + private static final class SettingsAndContentServer implements ImmediateRequestGenerator { + final ModuleConfigurationTests.ServerConfigResponder settings = new ModuleConfigurationTests.ServerConfigResponder(); + final List contentFetches = new CopyOnWriteArrayList<>(); + + /** + * A request maker that routes by endpoint. + * + * @return the request maker + */ + @Override + public ImmediateRequestI createImmediateRequestMaker() { + ImmediateRequestI settingsRequestMaker = settings.createImmediateRequestMaker(); + return (requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log) -> { + if (!CONTENT_ENDPOINT.equals(customEndpoint)) { + settingsRequestMaker.doWork(requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log); + return; + } + if (!networkingIsEnabled) { + callback.callback(null); + return; + } + contentFetches.add(requestData); + callback.callback(new JSONObject(NO_CONTENT_RESPONSE)); + }; + } + } + + /** + * One request as the local server saw it. + */ + private static final class Received { + final Map params; + final long receivedAtNs; + volatile long answeredAtNs; + + /** + * Records a request as it arrives. + * + * @param params the decoded request parameters + * @param receivedAtNs {@link System#nanoTime()} when it arrived + */ + Received(Map params, long receivedAtNs) { + this.params = params; + this.receivedAtNs = receivedAtNs; + } + } + + /** + * A local HTTP server standing in for the request queue endpoint: it accepts every request, holds + * back the answers a test asks it to, and records each request with the moment it arrived and the + * moment its answer started. One handler thread, so requests are recorded in the order they came. + */ + private static final class QueueServer { + final HttpServer http; + final List received = new CopyOnWriteArrayList<>(); + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private final Queue answerDelaysMs = new ConcurrentLinkedQueue<>(); + + /** + * Starts the server on a free loopback port. + * + * @throws IOException when the server cannot start + */ + QueueServer() throws IOException { + http = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + http.createContext("/", this::handle); + http.setExecutor(executor); + http.start(); + } + + /** + * Holds back the answers to the next requests, one delay per request in arrival order. Any + * request after those is answered right away. + * + * @param delaysMs how long to hold each answer back, in milliseconds + */ + void holdTheNextAnswers(long... delaysMs) { + for (long delayMs : delaysMs) { + answerDelaysMs.add(delayMs); + } + } + + /** + * Records a request, holds its answer back when asked to, then accepts it. + * + * @param exchange the request and its response + * @throws IOException when the response cannot be written + */ + private void handle(HttpExchange exchange) throws IOException { + String query = exchange.getRequestURI().getRawQuery(); + String body = new String(readAll(exchange.getRequestBody()), StandardCharsets.UTF_8); + Received request = new Received(parseParams(query != null && !query.isEmpty() ? query : body), System.nanoTime()); + received.add(request); + + Long delayMs = answerDelaysMs.poll(); + if (delayMs != null && delayMs > 0) { + try { + Thread.sleep(delayMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + byte[] response = "{\"result\":\"Success\"}".getBytes(StandardCharsets.UTF_8); + request.answeredAtNs = System.nanoTime(); + exchange.sendResponseHeaders(200, response.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response); + } + } + + /** + * The port the server listens on. + * + * @return the port + */ + int port() { + return http.getAddress().getPort(); + } + + /** + * Stops the server and its handler thread, even while it holds an answer back. + */ + void stop() { + http.stop(0); + executor.shutdownNow(); + } + } + + /** + * A display that shows nothing, so the content zone can be entered. + */ + private static final class FakeDisplay implements ContentDisplay { + /** + * A landscape screen. + * + * @return the screen + */ + @Override + public ContentScreen getScreen() { + return new ContentScreen(1600, 900); + } + + /** + * Shows nothing. + * + * @param content the content to show + * @param onClosed called when the content closes + */ + @Override + public void present(ContentData content, ContentCloseCallback onClosed) { + } + } + + /** + * Starts {@link #queueServer}. + * + * @throws IOException when the server cannot start + */ + private void startQueueServer() throws IOException { + queueServer = new QueueServer(); + } + + /** + * Initializes the SDK with {@link #server} answering its settings requests and content fetches. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + server.settings.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * A configuration pointing at {@link #queueServer}, with events sent one per request, no request + * cooldowns, and every log line kept. + * + * @return the configuration + */ + private Config networkedConfig() { + File directory = TestUtils.getTestSDirectory(); + TestUtils.checkSdkStorageRootDirectoryExist(directory); + return new Config("http://127.0.0.1:" + queueServer.port(), TestUtils.SERVER_APP_KEY, directory) + .setApplicationVersion(TestUtils.APPLICATION_VERSION) + .setCustomDeviceId(TestUtils.DEVICE_ID) + .enableFeatures(Config.Feature.Events) + .setEventQueueSizeToSend(1) + .setNetworkRequestCooldown(0) + .setNetworkImportantRequestCooldown(0) + .setLogListener((message, level) -> logs.add(message)); + } + + /** + * Records events one after another, letting the clock move on after each, so every request gets + * a storage ID of its own. + * + * @param keys the event keys + */ + private static void recordEvents(String... keys) { + for (String key : keys) { + Countly.instance().events().recordEvent(key); + TestUtils.letTheClockCatchUp(); + } + } + + /** + * Stores a request file before init, as an earlier run would have left it. + * + * @param id the storage ID, which names the file + * @param marker a parameter that tells the requests apart on the server + * @param timestamp the {@code timestamp} parameter, {@code null} for none + * @throws IOException when the file cannot be written + */ + private static void storeRequest(long id, String marker, Long timestamp) throws IOException { + Request request = new Request(id); + request.params = new Params("app_key", TestUtils.SERVER_APP_KEY, "device_id", TestUtils.DEVICE_ID, "marker", marker); + if (timestamp != null) { + request.params.add("timestamp", timestamp); + } + File directory = TestUtils.getTestSDirectory(); + TestUtils.checkSdkStorageRootDirectoryExist(directory); + Files.write(new File(directory, "[CLY]_request_" + id).toPath(), request.store(new Log(Config.LoggingLevel.OFF, null))); + } + + /** + * Moves the {@code timestamp} parameter of every stored request back by the given time, as if it + * had been recorded that much earlier, keeping its storage ID. + * + * @param byMs how far back, in milliseconds + * @throws IOException when a file cannot be rewritten + */ + private static void ageStoredRequests(long byMs) throws IOException { + Pattern timestamp = Pattern.compile("(^|&)timestamp=(\\d+)"); + for (File file : requestFiles()) { + String content = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + Matcher matcher = timestamp.matcher(content); + Assert.assertTrue("no timestamp in " + file.getName(), matcher.find()); + long aged = Long.parseLong(matcher.group(2)) - byMs; + String agedContent = content.substring(0, matcher.start(2)) + aged + content.substring(matcher.end(2)); + Files.write(file.toPath(), agedContent.getBytes(StandardCharsets.UTF_8)); + } + } + + /** + * The stored request files. + * + * @return the files, never {@code null} + */ + private static File[] requestFiles() { + File[] files = TestUtils.getTestSDirectory().listFiles((dir, name) -> name.startsWith("[CLY]_request_")); + return files == null ? new File[0] : files; + } + + /** + * Counts the stored requests without reading them, as the network loop can remove one while it + * is read. + * + * @return the number of stored requests + */ + private static int queuedRequestCount() { + return requestFiles().length; + } + + /** + * The storage IDs of the stored requests. + * + * @return the IDs + */ + private static Set storedRequestIds() { + Set ids = new HashSet<>(); + for (File file : requestFiles()) { + ids.add(Long.parseLong(file.getName().substring("[CLY]_request_".length()))); + } + return ids; + } + + /** + * How many requests the memory queue of backend mode holds. + * + * @return the number of requests + */ + private static int memoryQueueSize() { + SDKCore core = SDKCore.instance; + synchronized (core.lockBRQStorage) { + return core.requestQueueMemory.size(); + } + } + + /** + * The key of the event each request the server received carried, in the order they arrived. + * + * @return the event keys + */ + private List receivedEventKeys() { + List keys = new ArrayList<>(); + for (Received request : queueServer.received) { + JSONArray requestKeys = eventKeys(request.params); + for (int i = 0; i < requestKeys.length(); i++) { + keys.add(requestKeys.getString(i)); + } + } + return keys; + } + + /** + * One parameter of each request the server received, in the order they arrived. + * + * @param key the parameter + * @return its values + */ + private List receivedParams(String key) { + List values = new ArrayList<>(); + for (Received request : queueServer.received) { + values.add(request.params.get(key)); + } + return values; + } + + /** + * How long the server had to wait for a request after it started answering the one before it. + * + * @param index the position of the request, from 1 + * @return the gap in milliseconds + */ + private long gapBeforeRequestMs(int index) { + Received previous = queueServer.received.get(index - 1); + Received next = queueServer.received.get(index); + return (next.receivedAtNs - previous.answeredAtNs) / 1_000_000L; + } + + /** + * The keys of the events a request carries, in order. + * + * @param params the parameters of the request + * @return the keys as a JSON array, empty when the request carries no events + */ + private static JSONArray eventKeys(Map params) { + JSONArray keys = new JSONArray(); + String events = params.get("events"); + if (events == null) { + return keys; + } + JSONArray eventArray = new JSONArray(events); + for (int i = 0; i < eventArray.length(); i++) { + keys.put(eventArray.getJSONObject(i).getString("key")); + } + return keys; + } + + /** + * The live backoff threads. + * + * @return the threads + */ + private static Set backoffThreads() { + Set threads = new HashSet<>(); + for (Thread thread : Thread.getAllStackTraces().keySet()) { + if (DefaultNetworking.BACKOFF_THREAD_NAME.equals(thread.getName()) && thread.isAlive()) { + threads.add(thread); + } + } + return threads; + } + + /** + * Whether every given thread has ended. + * + * @param threads the threads + * @return {@code true} when none is alive + */ + private static boolean noneAlive(Set threads) { + for (Thread thread : threads) { + if (thread.isAlive()) { + return false; + } + } + return true; + } + + /** + * Holds the first tick of every zone timer an hour away, so the zone fetches only when a test + * ticks it by hand. + */ + private static void holdTheZoneTimer() { + contentModule().startedAtForTests(System.currentTimeMillis() + HOUR_MS); + } + + /** + * The content module of the running SDK. + * + * @return the module + */ + private static ModuleContent contentModule() { + return SDKCore.instance.module(ModuleContent.class); + } + + /** + * Ticks the zone once by hand and asserts that it fetched nothing, as no zone is active. + * + * @param fetchesSoFar the content fetches made so far + */ + private void assertZoneIsInactive(int fetchesSoFar) { + contentModule().onZoneTimerTick(); + Assert.assertEquals(fetchesSoFar, server.contentFetches.size()); + } + + /** + * Ticks the zone by hand until it fetches, which it does once a refresh entered it. + * + * @param expectedFetches the content fetches expected once it fetched + */ + private void awaitZoneFetch(int expectedFetches) throws InterruptedException { + boolean fetched = waitNudging(() -> { + contentModule().onZoneTimerTick(); + return server.contentFetches.size() >= expectedFetches; + }); + Assert.assertTrue("the content zone never fetched, logs: " + logs, fetched); + Assert.assertEquals(expectedFetches, server.contentFetches.size()); + } + + /** + * Waits until the log holds a number of lines containing a text. + * + * @param text the text + * @param count how many lines + */ + private void awaitLogs(String text, int count) throws InterruptedException { + Assert.assertTrue("never logged [" + text + "] " + count + " times, logs: " + logs, waitFor(() -> logsContaining(text) >= count)); + Assert.assertEquals(count, logsContaining(text)); + } + + /** + * How many log lines so far contain a text. + * + * @param text the text + * @return the number of lines + */ + private int logsContaining(String text) { + int count = 0; + for (String line : logs) { + if (line.contains(text)) { + count++; + } + } + return count; + } + + /** + * Makes the send loop look at the queue, as a newly stored request does. + */ + private static void nudge() { + SDKCore core = SDKCore.instance; + if (core != null && core.networking != null && core.config != null) { + core.networking.check(core.config); + } + } + + /** + * Nudges the send loop every {@link #POLL_MS} until the given moment. + * + * @param deadlineNs the moment, as {@link System#nanoTime()} + */ + private static void nudgeUntil(long deadlineNs) throws InterruptedException { + while (System.nanoTime() < deadlineNs) { + nudge(); + Thread.sleep(POLL_MS); + } + } + + /** + * Sleeps until the given moment. + * + * @param deadlineNs the moment, as {@link System#nanoTime()} + */ + private static void sleepUntil(long deadlineNs) throws InterruptedException { + long remainingMs = (deadlineNs - System.nanoTime()) / 1_000_000L; + if (remainingMs > 0) { + Thread.sleep(remainingMs); + } + } + + /** + * Polls until the condition holds or {@link #WAIT_BUDGET_MS} runs out, without nudging the send + * loop, so the SDK has to drain its queue on its own. + * + * @param condition the condition + * @return whether the condition holds + */ + private static boolean waitFor(BooleanSupplier condition) throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_BUDGET_MS; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return true; + } + Thread.sleep(POLL_MS); + } + return condition.getAsBoolean(); + } + + /** + * Polls like {@link #waitFor(BooleanSupplier)}, nudging the send loop on every round. + * + * @param condition the condition + * @return whether the condition holds + */ + private static boolean waitNudging(BooleanSupplier condition) throws InterruptedException { + return waitFor(() -> { + nudge(); + return condition.getAsBoolean(); + }); + } + + /** + * Decodes url encoded request parameters. + * + * @param data the parameters + * @return the parameters by name + */ + private static Map parseParams(String data) { + Map params = new HashMap<>(); + for (String pair : data.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] keyValue = pair.split("=", 2); + params.put(Utils.urldecode(keyValue[0]), keyValue.length > 1 ? Utils.urldecode(keyValue[1]) : ""); + } + return params; + } + + /** + * Reads a stream to its end. + * + * @param stream the stream + * @return the bytes + * @throws IOException when the stream cannot be read + */ + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = stream.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java new file mode 100644 index 00000000..2e69a4ef --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java @@ -0,0 +1,1290 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpServer; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.math.BigDecimal; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * SDK behavior settings, the server side configuration served by {@code /o/sdk?method=sc}: how + * {@link ModuleConfiguration} validates, stores and resolves them, and how the SDK applies + * the values it owns at init and when a response changes them. + *

+ * The settings request goes through a {@link ServerConfigResponder} injected on the + * {@link InternalConfig} before init. It answers on the calling thread, so the response of the fetch + * made at init is applied before init returns, and a later response is pushed by calling + * {@link ModuleConfiguration#fetchConfigFromServer(InternalConfig)} the way the refresh timer does. + */ +@RunWith(JUnit4.class) +public class ModuleConfigurationTests { + + private static final Set TOP_LEVEL_KEYS = names("v", "t", "c", "lg", "ct"); + + private ServerConfigResponder server; + + /** + * Starts every test from empty storage, without a test module and with a fresh responder. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + server = new ServerConfigResponder(); + } + + /** + * Stops the SDK, clearing its data, and removes the test module. + */ + @After + public void afterTest() { + Countly.instance().halt(); + SDKCore.testDummyModule = null; + } + + // region scenarios + + /** + * The type table of every settings key, run through the validation a response goes through. + *

+ * Verifies that each of the 42 keys keeps every valid value and loses every value of the wrong + * type or range, that unknown keys are dropped, and that a decimal, which the JSON parser hands + * over as a {@link BigDecimal}, is still accepted as {@code bom_rqp}. + */ + @Test + public void removeUnsupportedKeys_keepsEveryValidValueAndDropsTheRest() throws IllegalAccessException { + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings()); + ModuleConfiguration module = module(); + + Map> valid = new LinkedHashMap<>(); + Map> invalid = new LinkedHashMap<>(); + for (String key : Arrays.asList("tracking", "networking", "st", "vt", "cet", "ecz", "crt", "ast", "avt", "acr", "lt", "rcz", "bom", "log", "cr")) { + valid.put(key, Arrays.asList("true", "false")); + invalid.put(key, Arrays.asList("1", "0", "\"true\"", "null", "{}", "[]")); + } + for (String key : Arrays.asList("scui", "rqs", "eqs", "sui", "lkl", "lvs", "lsv", "lbc", "ltlpt", "ltl", "upcl", "bom_at", "bom_ra", "bom_d")) { + valid.put(key, Arrays.asList("1", "7", "2147483647")); + invalid.put(key, Arrays.asList("0", "-5", "1.5", "5.0", "\"5\"", "true", "3000000000", "null")); + } + valid.put("dort", Arrays.asList("0", "1", "72")); + invalid.put("dort", Arrays.asList("-1", "0.5", "\"1\"", "false")); + valid.put("czi", Arrays.asList("16", "300")); + invalid.put("czi", Arrays.asList("15", "0", "-16", "16.0", "\"16\"")); + valid.put("bom_rqp", Arrays.asList("0.5", "0.01", "0.99", "1e-1")); + invalid.put("bom_rqp", Arrays.asList("0", "1", "0.0", "1.0", "-0.5", "1.5", "\"0.5\"", "true", "null")); + for (String key : Arrays.asList("eb", "ew", "sb", "sw", "upb", "upw", "jte", "jtv")) { + valid.put(key, Arrays.asList("[]", "[\"a\",\"b\"]")); + invalid.put(key, Arrays.asList("\"a\"", "{}", "1", "true", "null")); + } + for (String key : Arrays.asList("esb", "esw")) { + valid.put(key, Arrays.asList("{}", "{\"purchase\":[\"card\"]}")); + invalid.put(key, Arrays.asList("[]", "\"a\"", "1", "null")); + } + + Set settingsKeys = new HashSet<>(settingsKeysOfTheModule()); + Assert.assertEquals("the table covers every settings key the module declares", settingsKeys, valid.keySet()); + Assert.assertEquals(42, settingsKeys.size()); + + for (Map.Entry> entry : valid.entrySet()) { + for (String literal : entry.getValue()) { + assertKeptByValidation(module, entry.getKey(), literal, true); + } + } + for (Map.Entry> entry : invalid.entrySet()) { + for (String literal : entry.getValue()) { + assertKeptByValidation(module, entry.getKey(), literal, false); + } + } + for (String unknownKey : Arrays.asList("heartbeat", "TRACKING", "someFutureFeature", "")) { + assertKeptByValidation(module, unknownKey, "true", false); + } + + JSONObject settings = validated(module, "{\"bom_rqp\":0.25,\"tracking\":\"no\",\"eqs\":3}"); + Assert.assertTrue(settings.get("bom_rqp") instanceof BigDecimal); + Assert.assertEquals(2, settings.length()); + + push(new JSONObject().put("v", 1).put("t", 1).put("c", new JSONObject("{\"bom_rqp\":0.25}")).toString()); + Assert.assertEquals(0.25, provider().getBOMRQPercentage(), 0.0); + } + + /** + * Every settings key at once, one key of each blacklist and whitelist pair, at a valid value that + * differs from the default, then every key at an invalid value. + *

+ * Verifies that each value reaches its provider getter and the store, that the request queue size + * is reported as coming from the settings, and that a response made only of invalid values stores + * no setting and brings every one back to the developer value, as one more change. + */ + @Test + public void allSettings_resolveFromAResponse_andInvalidValuesFallBackToTheDeveloperValues() throws IllegalAccessException { + AtomicInteger changes = installCountingModule().changes; + ServerConfigBuilder allKeys = new ServerConfigBuilder().allKeysAtNonDefaultValues(); + server.respondWith(allKeys); + init(TestUtils.getConfigSdkBehaviorSettings()); + + allKeys.validateAgainst(provider()); + Assert.assertTrue(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(1, changes.get()); + JSONObject storedSettings = storedConfig().getJSONObject("c"); + Assert.assertEquals(allKeys.config.keySet(), storedSettings.keySet()); + + ServerConfigBuilder allInvalid = new ServerConfigBuilder(); + for (String key : settingsKeysOfTheModule()) { + allInvalid.set(key, "tracking".equals(key) ? -1 : "not a valid value"); + } + push(allInvalid); + + ConfigurationProvider developer = ModuleConfiguration.developerDefaults(new InternalConfig(TestUtils.getConfigSdkBehaviorSettings())); + Set nameFilterKeys = names("eb", "ew", "upb", "upw", "sb", "sw", "esb", "esw"); + for (String key : allKeys.config.keySet()) { + if (!nameFilterKeys.contains(key)) { + Assert.assertEquals("value of '" + key + "'", ServerConfigBuilder.valueOf(developer, key), ServerConfigBuilder.valueOf(provider(), key)); + } + } + assertFilter(provider().getEventFilterList(), false, names()); + assertFilter(provider().getUserPropertyFilterList(), false, names()); + assertFilter(provider().getSegmentationFilterList(), false, names()); + Assert.assertTrue(provider().getEventSegmentationFilterList().getFilterList().isEmpty()); + Assert.assertFalse(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(2, changes.get()); + Assert.assertEquals(0, storedConfig().getJSONObject("c").length()); + } + + /** + * The envelope of a response: {@code v}, {@code t} and an object {@code c} are required, any + * other top level key is allowed. + *

+ * Verifies that a response missing one of the three, or with a {@code c} that is not an object, + * is rejected without touching the store, that an empty {@code c} is accepted, that extra top + * level keys are accepted quietly and never stored, so neither {@code lg} nor {@code ct} ever is + * while {@code ct} still asks for one connection test, and that a rejected response after an + * accepted one keeps the merged settings. + */ + @Test + public void envelope_requiresVersionTimestampAndSettingsObject_andOnlyThoseAreStored() { + List warnings = new CopyOnWriteArrayList<>(); + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings().setLogListener((message, level) -> { + if (level == Config.LoggingLevel.WARN || level == Config.LoggingLevel.ERROR) { + warnings.add(message); + } + })); + Assert.assertNull(storedConfig()); + + for (String rejected : Arrays.asList("{}", "{'t':2,'c':{'tracking':false}}", "{'v':1,'c':{'tracking':false}}", "{'v':1,'t':2}", + "{'v':1,'t':2,'c':123}", "{'v':1,'t':2,'c':false}", "{'v':1,'t':2,'c':'fdf'}", "{'v':1,'t':2,'c':[]}")) { + push(rejected); + Assert.assertNull("stored after " + rejected, storedConfig()); + Assert.assertTrue(provider().getTrackingEnabled()); + } + + push("{'v':1,'t':2,'c':{}}"); + Assert.assertTrue(new JSONObject("{'v':1,'t':2,'c':{}}").similar(storedConfig())); + + warnings.clear(); + List connectionTestRequests = new CopyOnWriteArrayList<>(); + //the connection test module would run its probe battery against the test server over the network + module().setConnectionTestListener(connectionTestRequests::add); + push(new ServerConfigBuilder().version(2).timestamp(3).tracking(false) + .logGatheringOn("gather_id", "ew", 20) + .connectionTest(1) + .topLevelKey("someFutureFeature", "a value this SDK has never heard of")); + + Assert.assertEquals(1, connectionTestRequests.size()); + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertEquals(ConfigurationProvider.LogGatheringState.GATHERING, provider().getLogGatheringState()); + JSONObject stored = storedConfig(); + Assert.assertEquals(names("v", "t", "c"), stored.keySet()); + Assert.assertEquals(2, stored.getInt("v")); + Assert.assertEquals(3, stored.getLong("t")); + Assert.assertFalse(stored.getJSONObject("c").getBoolean("tracking")); + for (String warning : warnings) { + Assert.assertFalse("warned about a top level key: " + warning, warning.contains("someFutureFeature") || warning.contains("[lg]") || warning.contains("[ct]")); + } + + push("{'t':4,'c':{'tracking':true}}"); + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertTrue(stored.similar(storedConfig())); + } + + /** + * The layers of a value: developer configuration, then the provided settings while nothing is + * stored, then the stored settings, then each server response. + *

+ * Verifies each layer overriding the one below it, that the provided settings are stored and + * ignored once something is stored, that a response replaces the stored settings, so a setting it + * leaves out goes back to the developer value, and that the replacement survives {@code stop()} + * and a new init. + */ + @Test + public void precedence_developerThenProvidedThenStoredThenServer_andAResponseReplacesTheStoredSettings() { + server.pending(); + init(developerConfig()); + + Assert.assertEquals(7, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(50, provider().getRequestQueueMaxSize()); + Assert.assertFalse(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(30, provider().getSessionUpdateInterval()); + Assert.assertEquals(40, provider().getMaxBreadcrumbCount()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider().getMaxKeyLength()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider().getUserPropertyCacheLimit()); + Assert.assertTrue(provider().getLoggingEnabled()); + Assert.assertNull(storedConfig()); + Countly.instance().halt(); + + init(developerConfig().setSdkBehaviorSettings(new ServerConfigBuilder().eventQueueSize(11).requestQueueSize(22).build())); + + Assert.assertEquals(11, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(22, provider().getRequestQueueMaxSize()); + Assert.assertTrue(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(30, provider().getSessionUpdateInterval()); + Assert.assertEquals(2, storedConfig().getJSONObject("c").length()); + + push(new ServerConfigBuilder().eventQueueSize(33).breadcrumbLimit(5).logging(false)); + + Assert.assertEquals(33, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(50, provider().getRequestQueueMaxSize()); + Assert.assertFalse(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(5, provider().getMaxBreadcrumbCount()); + Assert.assertEquals(30, provider().getSessionUpdateInterval()); + Assert.assertFalse(provider().getLoggingEnabled()); + JSONObject storedSettings = storedConfig().getJSONObject("c"); + Assert.assertEquals(names("eqs", "lbc", "log"), storedSettings.keySet()); + Countly.instance().stop(); + + server.failing(); + init(developerConfig().setSdkBehaviorSettings(new ServerConfigBuilder().eventQueueSize(99).sessionTracking(false).build())); + + Assert.assertEquals(33, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(50, provider().getRequestQueueMaxSize()); + Assert.assertEquals(5, provider().getMaxBreadcrumbCount()); + Assert.assertFalse(provider().getLoggingEnabled()); + Assert.assertTrue(provider().getSessionTrackingEnabled()); + Assert.assertTrue(storedSettings.similar(storedConfig().getJSONObject("c"))); + } + + /** + * The settings request itself. + *

+ * Verifies that init sends exactly one request, to {@code /o/sdk?} with {@code method=sc} and the + * parameters every request carries, that it goes out even while the settings forbid networking, + * that the configuration module is initialized once and sits at its fixed index ahead of the + * feature modules, and that with settings requests disabled nothing is requested while the + * provided settings still apply. + */ + @Test + public void fetch_requestsTheSettingsAtInitWithTheRequiredParams_andNeverWhenDisabled() { + List initLines = new CopyOnWriteArrayList<>(); + server.respondWith(new ServerConfigBuilder().networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().setLogListener((message, level) -> { + if (message.startsWith("[ModuleConfiguration] init,")) { + initLines.add(message); + } + })); + + Assert.assertEquals(1, server.requests.size()); + Assert.assertEquals("/o/sdk?", server.endpoints.get(0)); + Map params = server.requests.get(0); + Assert.assertEquals("sc", params.get("method")); + TestUtils.validateRequiredParams(params); + Assert.assertEquals(TestUtils.SDK_NAME, params.get("sdk_name")); + Assert.assertEquals(TestUtils.SDK_VERSION, params.get("sdk_version")); + Assert.assertTrue(server.networkingFlags.get(0)); + + Assert.assertEquals(1, initLines.size()); + Assert.assertSame(module(), SDKCore.instance.modules.get(-1)); + Assert.assertSame(module(), SDKCore.instance.config.getConfigurationProvider()); + Assert.assertEquals(4L * 60 * 60, module().serverConfigUpdateTimer.getTimerDelaySeconds()); + + Assert.assertFalse(provider().getNetworkingEnabled()); + push(new ServerConfigBuilder().networking(true)); + Assert.assertEquals(2, server.requests.size()); + Assert.assertTrue(server.networkingFlags.get(1)); + Assert.assertTrue(provider().getNetworkingEnabled()); + Countly.instance().halt(); + + server = new ServerConfigResponder(); + init(TestUtils.getConfigSdkBehaviorSettings() + .disableSdkBehaviorSettingsUpdates() + .setSdkBehaviorSettings(new ServerConfigBuilder().tracking(false).build())); + + Assert.assertTrue(server.requests.isEmpty()); + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertNull(module().serverConfigUpdateTimer); + Assert.assertEquals(ConfigurationProvider.LogGatheringState.NOT_GATHERING, provider().getLogGatheringState()); + } + + /** + * Backend mode keeps the settings inert. + *

+ * Verifies that with stored and provided settings present, backend mode requests nothing, serves + * the developer configuration, starts no refresh timer, decides against log gathering and leaves + * the stored settings as they are for a later run without backend mode. + */ + @Test + public void backendMode_keepsTheSettingsInert() { + server.respondWith(new ServerConfigBuilder().tracking(false).eventQueueSize(3).requestQueueSize(5)); + init(TestUtils.getConfigSdkBehaviorSettings()); + Assert.assertFalse(provider().getTrackingEnabled()); + JSONObject storedBefore = storedConfig(); + Countly.instance().stop(); + + server = new ServerConfigResponder().respondWith(new ServerConfigBuilder().networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings() + .enableBackendMode() + .setEventQueueSizeToSend(9) + .setRequestQueueMaxSize(40) + .setSdkBehaviorSettings(new ServerConfigBuilder().networking(false).build())); + + Assert.assertTrue(server.requests.isEmpty()); + Assert.assertTrue(provider().getTrackingEnabled()); + Assert.assertTrue(provider().getNetworkingEnabled()); + Assert.assertTrue(SDKCore.instance.config.getNetworkingEnabled()); + Assert.assertEquals(9, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(40, provider().getRequestQueueMaxSize()); + Assert.assertFalse(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(ConfigurationProvider.LogGatheringState.NOT_GATHERING, provider().getLogGatheringState()); + Assert.assertNull(module().serverConfigUpdateTimer); + Assert.assertTrue(storedBefore.similar(storedConfig())); + } + + /** + * The consent requirement is applied when the SDK initializes, and only then. + *

+ * Verifies that a {@code cr} arriving while the SDK runs is stored but leaves the running SDK + * without consent gating, that the stored value is in place before the modules are built on the + * next init, so the feature modules wait for consent while an integration that never asked for + * consent calls an events interface that ignores calls instead of throwing, and that removing + * consent stops feature modules but never the configuration module, which keeps applying + * responses. + */ + @Test + public void consentRequirement_appliesAtInitOnly_andConsentRemovalNeverStopsTheConfigurationModule() { + server.respondWith(new ServerConfigBuilder().consentRequired(true)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events)); + + Assert.assertTrue(provider().getConsentRequired()); + Assert.assertFalse(SDKCore.instance.config.requiresConsent()); + Assert.assertTrue(Countly.isTracking(Config.Feature.Events)); + Assert.assertNotNull(Countly.instance().events()); + Assert.assertTrue(storedConfig().getJSONObject("c").getBoolean("cr")); + Countly.instance().stop(); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events)); + + Assert.assertTrue(SDKCore.instance.config.requiresConsent()); + Assert.assertFalse(Countly.isTracking(Config.Feature.Events)); + Assert.assertSame(SDKCore.dormantEvents, Countly.instance().events()); + Countly.instance().events().recordEvent("before_consent"); + TestUtils.validateEQSize(0); + + Countly.onConsent(Config.Feature.Events); + Assert.assertTrue(Countly.isTracking(Config.Feature.Events)); + + ModuleConfiguration configurationModule = module(); + CountlyTimer refreshTimer = configurationModule.serverConfigUpdateTimer; + Assert.assertNotNull(refreshTimer); + + Countly.onConsentRemoval(Config.Feature.Events); + + Assert.assertFalse(Countly.isTracking(Config.Feature.Events)); + Assert.assertSame(configurationModule, SDKCore.instance.modules.get(-1)); + Assert.assertSame(configurationModule, SDKCore.instance.config.getConfigurationProvider()); + Assert.assertTrue(configurationModule.isActive()); + Assert.assertSame(refreshTimer, configurationModule.serverConfigUpdateTimer); + + push(new ServerConfigBuilder().consentRequired(false).tracking(false)); + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertFalse(provider().getConsentRequired()); + Assert.assertTrue(SDKCore.instance.config.requiresConsent()); + } + + /** + * The global timer follows the session update interval, and the refresh timer follows the + * update interval of the settings. + *

+ * Verifies that a stored {@code sui} starts the global timer at init, which then really ticks every + * second, that a changed {@code sui} restarts the timer and notifies every module once, that a + * response changing nothing restarts nothing and notifies nobody, that another change notifies the + * modules without touching the timer, and that a changed {@code scui} restarts the refresh timer. + */ + @Test + public void timers_followTheResolvedIntervals_andChangesAreDispatchedOnce() throws InterruptedException { + CountingModule counter = installCountingModule(); + AtomicInteger changes = counter.changes; + AtomicInteger ticks = counter.ticks; + + server.respondWith(new ServerConfigBuilder().sessionUpdateInterval(1)); + init(TestUtils.getConfigSdkBehaviorSettings()); + Assert.assertEquals(1, SDKCore.instance.countlyTimer.getTimerDelaySeconds()); + Assert.assertEquals(1, changes.get()); + Countly.instance().stop(); + + changes.set(0); + ticks.set(0); + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings()); + + CountlyTimer timerAtInit = SDKCore.instance.countlyTimer; + Assert.assertEquals(1, timerAtInit.getTimerDelaySeconds()); + Assert.assertEquals(0, changes.get()); + Assert.assertTrue("the global timer did not tick every second", waitFor(10_000, () -> ticks.get() >= 2)); + + ServerConfigBuilder slower = new ServerConfigBuilder().sessionUpdateInterval(3); + push(slower); + CountlyTimer restarted = SDKCore.instance.countlyTimer; + Assert.assertNotSame(timerAtInit, restarted); + Assert.assertEquals(3, restarted.getTimerDelaySeconds()); + Assert.assertEquals(1, changes.get()); + + push(slower); + Assert.assertSame(restarted, SDKCore.instance.countlyTimer); + Assert.assertEquals(1, changes.get()); + + push(new ServerConfigBuilder().sessionUpdateInterval(3).eventQueueSize(4)); + Assert.assertSame(restarted, SDKCore.instance.countlyTimer); + Assert.assertEquals(2, changes.get()); + + CountlyTimer refreshTimer = module().serverConfigUpdateTimer; + Assert.assertEquals(4L * 60 * 60, refreshTimer.getTimerDelaySeconds()); + push(new ServerConfigBuilder().serverConfigUpdateInterval(2)); + Assert.assertNotSame(refreshTimer, module().serverConfigUpdateTimer); + Assert.assertEquals(2L * 60 * 60, module().serverConfigUpdateTimer.getTimerDelaySeconds()); + Assert.assertEquals(3, changes.get()); + } + + /** + * {@code cet} covers custom events only, {@code tracking} covers every event. + *

+ * Verifies on the wire that with custom event tracking off a custom event, a {@code [CLY]_} key + * the SDK does not record itself included, is dropped while a view + * started through the views interface and an internal event still reach the queue, flushed at the + * event queue size the settings resolved, that tracking off then drops internal events too, and + * that custom events are recorded again once both are back on. Networking is off throughout, so + * the request stays on disk to be read. + */ + @Test + public void customEventTracking_off_dropsCustomEventsOnly_whileTrackingGatesEveryEvent() { + server.respondWith(new ServerConfigBuilder().customEventTracking(false).eventQueueSize(2).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events, Config.Feature.Views).setEventQueueSizeToSend(100)); + + Countly.instance().events().recordEvent("custom_event"); + Countly.instance().events().recordEvent("[CLY]_not_an_sdk_event"); + TestUtils.validateEQSize(0); + + Countly.instance().views().startView("home"); + TestUtils.validateEQSize(1); + Assert.assertEquals(ModuleViews.KEY_VIEW_EVENT, TestUtils.getCurrentEQ().get(0).key); + Assert.assertEquals(0, TestUtils.getCurrentRQ().length); + + Countly.instance().events().recordEvent("[CLY]_star_rating"); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + List sent = TestUtils.readEventsFromRequest(0, TestUtils.DEVICE_ID); + Assert.assertEquals(2, sent.size()); + Assert.assertEquals(ModuleViews.KEY_VIEW_EVENT, sent.get(0).key); + Assert.assertEquals("[CLY]_star_rating", sent.get(1).key); + TestUtils.validateEQSize(0); + + push(new ServerConfigBuilder().tracking(false)); + Countly.instance().events().recordEvent("[CLY]_star_rating"); + Countly.instance().views().startView("settings"); + TestUtils.validateEQSize(0); + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + + push(new ServerConfigBuilder().tracking(true).customEventTracking(true)); + Countly.instance().events().recordEvent("custom_event"); + TestUtils.validateEQSize(1); + Assert.assertEquals("custom_event", TestUtils.getCurrentEQ().get(0).key); + } + + /** + * Listing filters and journey trigger sets are published as immutable snapshots. + *

+ * Verifies every filter family and both journey sets as parsed, that a reader's snapshot never + * changes under it, that each response replaces every filter and journey set, so one it leaves out + * is turned off, and that the filters of the last response survive a restart. + */ + @Test + public void listingFilters_arePublishedAsSnapshots_andEachResponseReplacesThem() { + Map> perEvent = new LinkedHashMap<>(); + perEvent.put("purchase", names("card")); + server.respondWith(new ServerConfigBuilder() + .eventFilterList(names("blocked_a", "blocked_b"), false) + .segmentationFilterList(names("secret"), false) + .eventSegmentationFilterMap(perEvent, false) + .userPropertyFilterList(names("plan"), true) + .journeyTriggerEvents(names("checkout")) + .journeyTriggerViews(names("paywall"))); + init(TestUtils.getConfigSdkBehaviorSettings()); + + ConfigurationProvider.FilterList> events = provider().getEventFilterList(); + assertFilter(events, false, names("blocked_a", "blocked_b")); + assertFilter(provider().getSegmentationFilterList(), false, names("secret")); + assertFilter(provider().getUserPropertyFilterList(), true, names("plan")); + Assert.assertFalse(provider().getEventSegmentationFilterList().isWhitelist()); + Assert.assertEquals(perEvent, provider().getEventSegmentationFilterList().getFilterList()); + Assert.assertEquals(names("checkout"), provider().getJourneyTriggerEvents()); + Assert.assertEquals(names("paywall"), provider().getJourneyTriggerViews()); + assertUnmodifiable(events.getFilterList()); + assertUnmodifiable(provider().getEventSegmentationFilterList().getFilterList().get("purchase")); + assertUnmodifiable(provider().getJourneyTriggerEvents()); + try { + provider().getEventSegmentationFilterList().getFilterList().put("other", names("x")); + Assert.fail("the event segmentation filter can be changed by a reader"); + } catch (UnsupportedOperationException expected) { + Assert.assertEquals(1, provider().getEventSegmentationFilterList().getFilterList().size()); + } + + push(new ServerConfigBuilder().eventFilterList(names("allowed_event"), true)); + + assertFilter(events, false, names("blocked_a", "blocked_b")); + assertFilter(provider().getEventFilterList(), true, names("allowed_event")); + assertFilter(provider().getSegmentationFilterList(), false, names()); + assertFilter(provider().getUserPropertyFilterList(), false, names()); + Assert.assertTrue(provider().getEventSegmentationFilterList().getFilterList().isEmpty()); + Assert.assertTrue(provider().getJourneyTriggerEvents().isEmpty()); + Assert.assertTrue(provider().getJourneyTriggerViews().isEmpty()); + Assert.assertEquals(names("ew"), storedConfig().getJSONObject("c").keySet()); + + push(new ServerConfigBuilder().tracking(false)); + assertFilter(provider().getEventFilterList(), false, names()); + + push(new ServerConfigBuilder().segmentationFilterList(names("secret"), false).journeyTriggerEvents(names("checkout"))); + assertFilter(provider().getSegmentationFilterList(), false, names("secret")); + Assert.assertEquals(names("checkout"), provider().getJourneyTriggerEvents()); + Assert.assertTrue(provider().getTrackingEnabled()); + Assert.assertEquals(names("sb", "jte"), storedConfig().getJSONObject("c").keySet()); + Countly.instance().stop(); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings()); + assertFilter(provider().getSegmentationFilterList(), false, names("secret")); + Assert.assertEquals(names("checkout"), provider().getJourneyTriggerEvents()); + Assert.assertTrue(provider().getJourneyTriggerViews().isEmpty()); + Assert.assertTrue(provider().getTrackingEnabled()); + } + + /** + * A setting the server stops sending goes back to the developer value, or to the SDK default when + * the developer sets none, instead of keeping the last server value. + *

+ * Verifies with a developer configuration that sets the event and request queue sizes and the + * breadcrumb limit that a response leaving out tracking, the event queue size, the key length + * limit and an event blacklist brings back the default tracking, the developer's queue size, no + * key length limit and no event filter, that only what that response carries is stored, and that + * a response with no settings, which the server sends once every setting is removed, brings back + * every developer value and stores no setting. + */ + @Test + public void removedSettings_goBackToTheDeveloperValues_andAnEmptyResponseBringsBackAll() { + server.respondWith(new ServerConfigBuilder().tracking(false).eventQueueSize(3).keyLengthLimit(20).breadcrumbLimit(5) + .requestQueueSize(22).eventFilterList(names("blocked_event"), false)); + init(developerConfig()); + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertEquals(3, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(20, provider().getMaxKeyLength()); + assertFilter(provider().getEventFilterList(), false, names("blocked_event")); + + push(new ServerConfigBuilder().networking(true).requestQueueSize(22).breadcrumbLimit(5)); + Assert.assertTrue(provider().getTrackingEnabled()); + Assert.assertEquals(7, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(ModuleConfiguration.NO_LIMIT, provider().getMaxKeyLength()); + assertFilter(provider().getEventFilterList(), false, names()); + Assert.assertEquals(22, provider().getRequestQueueMaxSize()); + Assert.assertEquals(5, provider().getMaxBreadcrumbCount()); + Assert.assertEquals(names("networking", "rqs", "lbc"), storedConfig().getJSONObject("c").keySet()); + + push("{'v':1,'t':3,'c':{}}"); + Assert.assertEquals(50, provider().getRequestQueueMaxSize()); + Assert.assertFalse(provider().isRequestQueueMaxSizeFromBehaviorSettings()); + Assert.assertEquals(40, provider().getMaxBreadcrumbCount()); + Assert.assertTrue(provider().getLoggingEnabled()); + Assert.assertEquals(0, storedConfig().getJSONObject("c").length()); + } + + /** + * Settings provided at init stand in for the server's until its first response, which replaces + * them. + *

+ * Verifies that the provided settings apply at init, and that the first response, carrying only + * the request queue size, brings tracking and the event queue size back to the developer values + * and is all that is stored. + */ + @Test + public void providedSettings_applyUntilTheFirstResponse() { + server.pending(); + init(developerConfig().setSdkBehaviorSettings(new ServerConfigBuilder().tracking(false).eventQueueSize(11).build())); + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertEquals(11, provider().getEventQueueSizeThreshold()); + + push(new ServerConfigBuilder().requestQueueSize(500)); + Assert.assertTrue(provider().getTrackingEnabled()); + Assert.assertEquals(7, provider().getEventQueueSizeThreshold()); + Assert.assertEquals(500, provider().getRequestQueueMaxSize()); + Assert.assertEquals(names("rqs"), storedConfig().getJSONObject("c").keySet()); + } + + /** + * The settings can add a consent requirement but never lift the one the developer set. + *

+ * Verifies with {@link Config#setRequiresConsent(boolean)} on that a provided, a live and a stored + * {@code cr} false all leave consent required, so the feature modules wait for consent, and that + * without the developer's requirement a {@code cr} true is lifted again by a response without it. + */ + @Test + public void consentRequirement_aFalseFromTheSettingsNeverLiftsTheDevelopersRequirement() { + server.respondWith(new ServerConfigBuilder().consentRequired(false)); + init(TestUtils.getConfigSdkBehaviorSettings().setRequiresConsent(true).enableFeatures(Config.Feature.Events) + .setSdkBehaviorSettings(new ServerConfigBuilder().consentRequired(false).build())); + + Assert.assertTrue(provider().getConsentRequired()); + Assert.assertTrue(SDKCore.instance.config.requiresConsent()); + Assert.assertFalse(Countly.isTracking(Config.Feature.Events)); + Assert.assertFalse(storedConfig().getJSONObject("c").getBoolean("cr")); + Countly.instance().stop(); + + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings().setRequiresConsent(true).enableFeatures(Config.Feature.Events)); + Assert.assertTrue(provider().getConsentRequired()); + Assert.assertTrue(SDKCore.instance.config.requiresConsent()); + Assert.assertFalse(Countly.isTracking(Config.Feature.Events)); + Countly.instance().halt(); + + server = new ServerConfigResponder().respondWith(new ServerConfigBuilder().consentRequired(true)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events)); + Assert.assertTrue(provider().getConsentRequired()); + push(new ServerConfigBuilder().tracking(true)); + Assert.assertFalse(provider().getConsentRequired()); + } + + /** + * A journey trigger that a device ID change without merge ends is sent with the other events of + * the previous device ID, under that ID. + *

+ * Verifies with {@code checkout} as a journey trigger and networking off that ending a timed + * {@code checkout} on the change does not send the queue under the new ID: the queued event and + * the trigger are stored in one request that carries the previous device ID, and that request + * refreshes no content zone for the new device ID. + */ + @Test + public void journeyTrigger_endedByADeviceIdChange_isSentUnderThePreviousDeviceId() { + server.respondWith(new ServerConfigBuilder().journeyTriggerEvents(names("checkout")).eventQueueSize(100).networking(false)); + init(TestUtils.getConfigSdkBehaviorSettings().enableFeatures(Config.Feature.Events)); + + Countly.instance().events().recordEvent("old_user_event"); + Countly.instance().events().startEvent("checkout"); + Countly.instance().deviceId().changeWithoutMerge("new_device_id"); + TestUtils.letTheClockCatchUp(); + + Assert.assertEquals(1, TestUtils.getCurrentRQ().length); + List events = TestUtils.readEventsFromRequest(0, TestUtils.DEVICE_ID); + Assert.assertEquals(2, events.size()); + Assert.assertEquals("old_user_event", events.get(0).key); + Assert.assertEquals("checkout", events.get(1).key); + Assert.assertTrue(SDKCore.instance.module(ModuleEvents.class).journeyTriggerRequestIds.isEmpty()); + } + + /** + * The {@code ct} connection test flag of a live response. + *

+ * Verifies the truthiness table of the flag and that it is removed from the response, then through + * live fetches that a truthy flag reaches the registered listener once, after the settings of the + * same response were applied, with the latency measured from just before the request to its + * response, that a falsy flag does not, that a response whose settings are rejected can still ask, + * and that the flag is never stored. + */ + @Test + public void connectionTestFlag_isReadByValue_strippedFromTheResponse_andHandedOnWithTheFetchLatency() { + Object[][] table = { + { "true", true }, { "false", false }, { "1", true }, { "0", false }, { "0.0", false }, { "2.5", true }, { "-1", true }, + { "\"1\"", true }, { "\"0\"", false }, { "\"false\"", false }, { "\"FALSE\"", false }, { "\" false \"", false }, + { "\"\"", false }, { "\" \"", false }, { "\"yes\"", true }, { "\"true\"", true }, { "null", false }, + { "{}", true }, { "[]", true }, { "{\"a\":1}", true } + }; + for (Object[] row : table) { + JSONObject response = new JSONObject("{\"v\":1,\"t\":1,\"c\":{},\"ct\":" + row[0] + "}"); + Assert.assertEquals("ct = " + row[0], row[1], ModuleConfiguration.extractConnectionTestFlag(response)); + Assert.assertFalse("ct = " + row[0] + " was left in the response", response.has("ct")); + } + Assert.assertFalse(ModuleConfiguration.extractConnectionTestFlag(new JSONObject("{\"v\":1,\"t\":1,\"c\":{}}"))); + Assert.assertFalse(ModuleConfiguration.extractConnectionTestFlag(null)); + + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings()); + List latencies = new CopyOnWriteArrayList<>(); + List trackingWhenNotified = new CopyOnWriteArrayList<>(); + module().setConnectionTestListener(fetchLatencyMs -> { + latencies.add(fetchLatencyMs); + trackingWhenNotified.add(provider().getTrackingEnabled()); + }); + + server.delayedBy(80); + push(new ServerConfigBuilder().tracking(false).connectionTest(1)); + Assert.assertEquals(1, latencies.size()); + Assert.assertTrue("latency " + latencies.get(0), latencies.get(0) >= 70); + Assert.assertEquals(Arrays.asList(false), trackingWhenNotified); + + server.delayedBy(0); + push(new ServerConfigBuilder().connectionTest(0)); + push(new ServerConfigBuilder().connectionTest("false")); + push(new ServerConfigBuilder()); + Assert.assertEquals(1, latencies.size()); + + push("{\"ct\":true}"); + Assert.assertEquals(2, latencies.size()); + Assert.assertFalse(storedConfig().has("ct")); + + module().setConnectionTestListener(null); + push(new ServerConfigBuilder().connectionTest(1).tracking(true)); + Assert.assertTrue(provider().getTrackingEnabled()); + Assert.assertEquals(2, latencies.size()); + } + + /** + * The {@code lg} directive of a live response turns log gathering on and off. + *

+ * Verifies that a directive with a gather id arms gathering with its levels and batch size and is + * never stored, that {@code "e":false} and a response without any directive both decide off and + * drop the gather id, and that a failed fetch after a decision changes nothing. + */ + @Test + public void logGatheringDirective_enabledThenDisabled_isAppliedBothWays() { + server.respondWith(new ServerConfigBuilder().logGatheringOn("gather_id_3", "ewd", 40)); + init(TestUtils.getConfigSdkBehaviorSettings()); + + assertGathering("gather_id_3", "ewd", 40); + Assert.assertFalse(storedConfig().has("lg")); + + push(new ServerConfigBuilder().logGatheringOff()); + assertNotGathering(); + + push(new ServerConfigBuilder().logGatheringOn("gather_id_3", "ewd", 40)); + assertGathering("gather_id_3", "ewd", 40); + + push(new ServerConfigBuilder().tracking(false)); + assertNotGathering(); + Assert.assertFalse(provider().getTrackingEnabled()); + + push(new ServerConfigBuilder().logGatheringOn("gather_id_4", "e", 20)); + server.failing(); + module().fetchConfigFromServer(SDKCore.instance.config); + assertGathering("gather_id_4", "e", 20); + } + + /** + * A directive that enables gathering without a usable gather id, or with {@code e} that is not a + * boolean {@code true}, decides off. A working directive is applied between the cases, so no + * assertion passes because the state was already off. + */ + @Test + public void logGatheringDirective_withoutUsableGatherId_doesNotEnableGathering() { + server.respondWith(new ServerConfigBuilder().logGatheringOn("gather_id_4", "ew", 20)); + init(TestUtils.getConfigSdkBehaviorSettings()); + assertGathering("gather_id_4", "ew", 20); + + List unusable = Arrays.asList( + new ServerConfigBuilder().logGatheringDirective(true, null, "ew", 20), + new ServerConfigBuilder().logGatheringDirective(true, " ", "ew", 20), + new ServerConfigBuilder().logGatheringDirective(true, 42, "ew", 20), + new ServerConfigBuilder().logGatheringDirective("true", "gather_id_4", "ew", 20), + new ServerConfigBuilder().logGatheringDirective(1, "gather_id_4", "ew", 20), + new ServerConfigBuilder().topLevelKey("lg", "on")); + for (ServerConfigBuilder directive : unusable) { + push(directive); + assertNotGathering(); + + push(new ServerConfigBuilder().logGatheringOn("gather_id_4", "ew", 20)); + assertGathering("gather_id_4", "ew", 20); + } + + push(new ServerConfigBuilder().logGatheringOn(" gather_id_5 ", "ew", 20)); + assertGathering("gather_id_5", "ew", 20); + } + + /** + * The levels and the batch size of a directive are sanitized: unknown level characters go, the + * rest is lower cased and deduplicated in the order sent, nothing usable falls back to every level, + * and the batch size is clamped into [10, 500] or falls back to 100 when it is not a number. + */ + @Test + public void logGatheringDirective_levelsAndBatchSize_areSanitized() { + server.respondWith(new ServerConfigBuilder().logGatheringOn("gather_id_5", "xyz!", 100)); + init(TestUtils.getConfigSdkBehaviorSettings()); + assertGathering("gather_id_5", ModuleConfiguration.logGatheringAllLevels, 100); + + Object[][] steps = { + { "ew", 100, "ew", 100 }, + { null, 100, ModuleConfiguration.logGatheringAllLevels, 100 }, + { "ew", 100, "ew", 100 }, + { 5, 100, ModuleConfiguration.logGatheringAllLevels, 100 }, + { "EWID", 100, "ewid", 100 }, + { "wweeww", 100, "we", 100 }, + { "vq d", 100, "vd", 100 }, + { "ew", 1, "ew", ModuleConfiguration.logGatheringMinBatchSize }, + { "ew", 9999, "ew", ModuleConfiguration.logGatheringMaxBufferedLines }, + { "ew", 250, "ew", 250 }, + { "ew", "250", "ew", ModuleConfiguration.logGatheringDefaultBatchSize }, + { "ew", 250, "ew", 250 }, + { "ew", null, "ew", ModuleConfiguration.logGatheringDefaultBatchSize }, + { "ew", 37.9, "ew", 37 } + }; + for (Object[] step : steps) { + push(new ServerConfigBuilder().logGatheringDirective(true, "gather_id_5", step[0], step[1])); + assertGathering("gather_id_5", (String) step[2], (Integer) step[3]); + } + } + + /** + * Only a live response decides log gathering. + *

+ * Verifies that a stored directive, even an armed one, leaves the state undecided and is dropped + * from the stored copy, that a stored configuration without one also stays undecided, that a + * failed fetch decides off while undecided, and that disabled settings requests decide off at + * init even with an armed directive in the provided settings. + */ + @Test + public void logGathering_isUndecidedUntilALiveResponse_failedFetchAndDisabledRequestsDecideOff() { + TestUtils.writeToFile(SDKStorage.JSON_FILE_NAME, new JSONObject() + .put("sc", new ServerConfigBuilder().tracking(false).logGatheringOn("stored_gather_id", "ewidv", 40).build()).toString()); + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings()); + + Assert.assertFalse(provider().getTrackingEnabled()); + Assert.assertEquals(ConfigurationProvider.LogGatheringState.UNDECIDED, provider().getLogGatheringState()); + Assert.assertNull(provider().getLogGatheringId()); + Assert.assertEquals(ModuleConfiguration.logGatheringAllLevels, provider().getLogGatheringLevels()); + Assert.assertEquals(ModuleConfiguration.logGatheringDefaultBatchSize, provider().getLogGatheringBatchSize()); + Assert.assertFalse(storedConfig().has("lg")); + Countly.instance().stop(); + + init(TestUtils.getConfigSdkBehaviorSettings()); + Assert.assertEquals(ConfigurationProvider.LogGatheringState.UNDECIDED, provider().getLogGatheringState()); + Countly.instance().stop(); + + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings()); + assertNotGathering(); + Countly.instance().halt(); + + server = new ServerConfigResponder(); + init(TestUtils.getConfigSdkBehaviorSettings() + .disableSdkBehaviorSettingsUpdates() + .setSdkBehaviorSettings(new ServerConfigBuilder().logGatheringOn("provided_gather_id", "ew", 20).build())); + assertNotGathering(); + Assert.assertTrue(server.requests.isEmpty()); + } + + /** + * A queue held back by {@code networking: false} drains as soon as a response allows networking + * again, without waiting for another request or for the global timer. + *

+ * Verifies against a local server that nothing is sent while networking is off and that both held + * requests arrive once it is back on. + */ + @Test + public void networking_turningBackOn_drainsTheRequestsItHeldBack() throws Exception { + AtomicInteger received = new AtomicInteger(); + HttpServer httpServer = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + httpServer.createContext("/", exchange -> { + if (exchange.getRequestURI().getPath().startsWith("/i")) { + received.incrementAndGet(); + } + byte[] body = "{\"result\":\"Success\"}".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(body); + } + }); + httpServer.start(); + + try { + Config config = new Config("http://localhost:" + httpServer.getAddress().getPort(), TestUtils.SERVER_APP_KEY, TestUtils.getTestSDirectory()) + .setApplicationVersion(TestUtils.APPLICATION_VERSION) + .setCustomDeviceId(TestUtils.DEVICE_ID) + .enableFeatures(Config.Feature.Events) + .setEventQueueSizeToSend(1) + .setSdkBehaviorSettings(new ServerConfigBuilder().networking(false).build()); + server.pending(); + init(config); + + Countly.instance().events().recordEvent("held_1"); + Countly.instance().events().recordEvent("held_2"); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + //a queue that ignored the switch would have sent the first request well within this + Thread.sleep(1000); + Assert.assertEquals(0, received.get()); + Assert.assertEquals(2, TestUtils.getCurrentRQ().length); + + push(new ServerConfigBuilder().networking(true)); + + Assert.assertTrue("the held requests never drained", waitFor(30_000, () -> received.get() >= 2 && TestUtils.getCurrentRQ().length == 0)); + Assert.assertEquals(2, received.get()); + } finally { + Countly.instance().halt(); + httpServer.stop(0); + } + } + + // endregion + // region helpers + + /** + * Stands in for the settings endpoint of the server: records every immediate request the SDK + * makes and answers it on the calling thread with the response set at that moment. + */ + static final class ServerConfigResponder implements ImmediateRequestGenerator { + final List> requests = new CopyOnWriteArrayList<>(); + final List endpoints = new CopyOnWriteArrayList<>(); + final List networkingFlags = new CopyOnWriteArrayList<>(); + private volatile String response = null; + private volatile boolean answering = true; + private volatile long delayMs = 0; + + /** + * Answers every request with the given payload. + * + * @param builder the payload + * @return this responder + */ + ServerConfigResponder respondWith(ServerConfigBuilder builder) { + return respondWith(builder.build()); + } + + /** + * Answers every request with the given JSON text. + * + * @param json the response + * @return this responder + */ + ServerConfigResponder respondWith(String json) { + response = json; + answering = true; + return this; + } + + /** + * Answers every request as a failed one. + * + * @return this responder + */ + ServerConfigResponder failing() { + response = null; + answering = true; + return this; + } + + /** + * Never answers, as a server that has not answered yet. + * + * @return this responder + */ + ServerConfigResponder pending() { + answering = false; + return this; + } + + /** + * Waits before answering. + * + * @param delayMs how long, in milliseconds + * @return this responder + */ + ServerConfigResponder delayedBy(long delayMs) { + this.delayMs = delayMs; + return this; + } + + /** + * A request maker that records the request and answers it. + * + * @return the request maker + */ + @Override + public ImmediateRequestI createImmediateRequestMaker() { + return (requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log) -> { + requests.add(parseParams(requestData)); + endpoints.add(customEndpoint); + networkingFlags.add(networkingIsEnabled); + if (!answering) { + return; + } + if (delayMs > 0) { + try { + Thread.sleep(delayMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + String current = response; + callback.callback(current == null ? null : new JSONObject(current)); + }; + } + + /** + * Decodes url encoded request parameters. + * + * @param requestData the parameters + * @return the parameters by name + */ + private static Map parseParams(String requestData) { + Map params = new HashMap<>(); + for (String pair : requestData.split("&")) { + String[] keyValue = pair.split("=", 2); + params.put(Utils.urldecode(keyValue[0]), keyValue.length > 1 ? Utils.urldecode(keyValue[1]) : ""); + } + return params; + } + } + + /** + * Initializes the SDK with {@link #server} answering its settings requests. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + push(builder.build()); + } + + /** + * Answers the next settings request with the given JSON text and makes the SDK fetch. + * + * @param json the response + */ + private void push(String json) { + server.respondWith(json); + module().fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The configuration module of the running SDK. + * + * @return the module + */ + private static ModuleConfiguration module() { + return SDKCore.instance.module(ModuleConfiguration.class); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * The settings as stored on disk. + * + * @return the stored settings, {@code null} when nothing is stored + */ + private static JSONObject storedConfig() { + String stored = TestUtils.readJsonFile(SDKStorage.JSON_FILE_NAME).optString("sc", null); + return stored == null ? null : new JSONObject(stored); + } + + /** + * A developer configuration that sets every value the precedence test follows. + * + * @return the configuration + */ + private static Config developerConfig() { + return TestUtils.getConfigSdkBehaviorSettings() + .setEventQueueSizeToSend(7) + .setRequestQueueMaxSize(50) + .setUpdateSessionTimerDelay(30) + .setMaxBreadcrumbCount(40) + .setLoggingLevel(Config.LoggingLevel.ERROR); + } + + /** + * A test module that counts the ticks of the global timer and the configuration changes the SDK + * dispatches to every module. + */ + private static final class CountingModule extends ModuleBase { + final AtomicInteger changes = new AtomicInteger(); + final AtomicInteger ticks = new AtomicInteger(); + + /** + * Counts a tick of the global timer. + */ + @Override + protected void onTimer() { + ticks.incrementAndGet(); + } + + /** + * Counts a dispatched configuration change. + * + * @param config configuration of the running SDK + */ + @Override + protected void onSdkConfigurationChanged(InternalConfig config) { + changes.incrementAndGet(); + } + } + + /** + * Installs a {@link CountingModule} for the next init. + * + * @return the module + */ + private static CountingModule installCountingModule() { + CountingModule module = new CountingModule(); + SDKCore.testDummyModule = module; + return module; + } + + /** + * Every settings key the module declares, read off its {@code keyR} constants. + * + * @return the settings keys, without the top level keys + */ + private static List settingsKeysOfTheModule() throws IllegalAccessException { + List keys = new ArrayList<>(); + for (Field field : ModuleConfiguration.class.getDeclaredFields()) { + if (field.getName().startsWith("keyR") && Modifier.isStatic(field.getModifiers())) { + String key = (String) field.get(null); + if (!TOP_LEVEL_KEYS.contains(key)) { + keys.add(key); + } + } + } + return keys; + } + + /** + * Runs a settings object through the validation of a response. + * + * @param module the module that validates + * @param settings the settings object, as JSON text + * @return the settings left after validation + */ + private static JSONObject validated(ModuleConfiguration module, String settings) { + JSONObject envelope = new JSONObject("{\"v\":1,\"t\":1,\"c\":" + settings + "}"); + Assert.assertTrue(module.validateServerConfig(envelope)); + return envelope.getJSONObject("c"); + } + + /** + * Asserts whether validation keeps one setting. + * + * @param module the module that validates + * @param key the settings key + * @param literal the value, as JSON text + * @param kept whether the setting must be kept + */ + private static void assertKeptByValidation(ModuleConfiguration module, String key, String literal, boolean kept) { + JSONObject settings = validated(module, "{\"" + key + "\":" + literal + "}"); + Assert.assertEquals("[" + key + "] = " + literal, kept, settings.has(key)); + } + + /** + * Asserts the kind and the names of a filter. + * + * @param filter the filter + * @param isWhitelist the expected kind + * @param expected the expected names + */ + private static void assertFilter(ConfigurationProvider.FilterList> filter, boolean isWhitelist, Set expected) { + Assert.assertEquals(isWhitelist, filter.isWhitelist()); + Assert.assertEquals(expected, filter.getFilterList()); + } + + /** + * Asserts that a reader cannot change a collection. + * + * @param collection the collection + */ + private static void assertUnmodifiable(Collection collection) { + try { + collection.add("added_by_a_reader"); + Assert.fail("a reader can change " + collection); + } catch (UnsupportedOperationException expected) { + Assert.assertFalse(collection.contains("added_by_a_reader")); + } + } + + /** + * Asserts that log gathering is on with the given directive. + * + * @param gatherId the gather id + * @param levels the levels + * @param batchSize the batch size + */ + private static void assertGathering(String gatherId, String levels, int batchSize) { + Assert.assertEquals(ConfigurationProvider.LogGatheringState.GATHERING, provider().getLogGatheringState()); + Assert.assertEquals(gatherId, provider().getLogGatheringId()); + Assert.assertEquals(levels, provider().getLogGatheringLevels()); + Assert.assertEquals(batchSize, provider().getLogGatheringBatchSize()); + } + + /** + * Asserts that log gathering was decided off and nothing of a previous directive is left. + */ + private static void assertNotGathering() { + Assert.assertEquals(ConfigurationProvider.LogGatheringState.NOT_GATHERING, provider().getLogGatheringState()); + Assert.assertNull(provider().getLogGatheringId()); + Assert.assertEquals(ModuleConfiguration.logGatheringAllLevels, provider().getLogGatheringLevels()); + Assert.assertEquals(ModuleConfiguration.logGatheringDefaultBatchSize, provider().getLogGatheringBatchSize()); + } + + /** + * Polls until the condition holds or the budget runs out. + * + * @param timeoutMs the budget + * @param condition the condition + * @return whether the condition holds + */ + private static boolean waitFor(long timeoutMs, BooleanSupplier condition) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return true; + } + Thread.sleep(50); + } + return condition.getAsBoolean(); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConnectionTestTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConnectionTestTests.java new file mode 100644 index 00000000..2d7cbc0f --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConnectionTestTests.java @@ -0,0 +1,1432 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.net.ssl.HttpsURLConnection; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import ly.count.sdk.java.internal.ModuleConnectionTest.ProbeOutcome; +import ly.count.sdk.java.internal.ModuleConnectionTest.ResultRow; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ModuleConnectionTest.keyResults; +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; + +/** + * The connection test a live SDK behavior settings response arms with its top level {@code ct} + * flag: one battery of parameterless GET probes against every endpoint the SDK depends on, graded + * and queued as one {@code ct_results} report. + *

+ * Settings requests go through a {@link ModuleConfigurationTests.ServerConfigResponder}, which + * answers on the calling thread, so a response to the fetch made at init arms the test before init + * returns. Probes go to a local HTTP server that answers every path the way a Countly server behind + * the path prefix {@value #PREFIX} does. It refuses the SDK's own requests, which carry the app key, + * so the report stays in the request queue, where {@link TestUtils#getCurrentRQ()} reads it. A test + * that needs short deadlines or a fake probe transport sets them through the test hooks before it + * arms the test; no wait relies on the 10 second probe timeout or on the 60 second SDK timer. + */ +@RunWith(JUnit4.class) +public class ModuleConnectionTestTests { + + private static final String PREFIX = "/countly"; + private static final String[] EXPECTED_ORDER = { + "core", "core-write", "sc", "rc", "ab", "feedback", "feedback-widget", "feedback-submit", "content", "feedback-page", "feedback-assets", "content-page" + }; + private static final List EXPECTED_PATHS = Arrays.asList( + "/o/ping", "/i", "/o/sdk?method=rc", "/o/sdk?method=ab_fetch_variants", "/o/sdk?method=feedback", + "/o/surveys/nps/widget", "/o/surveys/survey/widget", "/o/feedback/widget", "/i/feedback/inputs", "/o/sdk/content", + "/feedback/nps", "/feedback/survey", "/feedback/rating", "/surveys/images/ct-probe.png", "/star-rating/images/ct-probe.png", + "/_external/content/"); + private static final Set ROW_KEYS = names("f", "ok", "st", "ms", "e", "n"); + private static final String MISSING_PARAMETER = "{\"result\":\"Missing parameter \\\"app_key\\\" or \\\"device_id\\\"\"}"; + private static final String NOT_RUN = "not run: battery cap"; + + private static final String BATTERY_STARTED = "[ModuleConnectionTest] startBattery, the server asked for a connection test"; + private static final String ALREADY_RUNNING = "[ModuleConnectionTest] startBattery, a battery is already running"; + private static final String NETWORKING_DISABLED = "[ModuleConnectionTest] startBattery, networking is disabled"; + private static final String REPORT_DROPPED = "[ModuleConnectionTest] runBatteryAndReport, the SDK stopped during the battery, dropping the report"; + + private static final long WAIT_BUDGET_MS = 30_000; + private static final long POLL_MS = 20; + + private ModuleConfigurationTests.ServerConfigResponder settings; + private ProbeServer server; + private final List logs = new CopyOnWriteArrayList<>(); + + /** + * Starts every test from empty storage, without a test module, with a fresh settings responder + * and a fresh local server. + * + * @throws IOException when the local server cannot start + */ + @Before + public void beforeTest() throws IOException { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + settings = new ModuleConfigurationTests.ServerConfigResponder(); + server = new ProbeServer(); + logs.clear(); + } + + /** + * Stops the SDK, clearing its data, then the local server. + */ + @After + public void afterTest() { + Countly.instance().halt(); + server.stop(); + } + + // region scenarios + + /** + * A healthy server behind a path prefix, armed by the response to the fetch made at init. + *

+ * Verifies that the battery reports every row reachable, in table order, with the status a + * Countly server answers a bare request with; that the report is one queued request of its own + * carrying the SDK identity and the required parameters and nothing else; that each of the 16 + * probes is a bare GET resolved against the prefix, with a single slash at the join although + * the configured URL ends in slashes, carrying only the method selector, the marker and the + * cache buster, no body and the custom request headers; that the flag is never stored while the + * rest of the response applies; and that the report then rides the queue to the server. + */ + @Test + public void armed_healthyServer_reportsEveryRowReachable_fromBareProbesResolvedAgainstThePathPrefix() throws Exception { + Map headers = new HashMap<>(); + headers.put("X-Countly-Test", "yes"); + settings.respondWith(new ServerConfigBuilder().viewTracking(false).connectionTest(1)); + init(config(server.url(PREFIX + "//")).addCustomNetworkRequestHeaders(headers)); + awaitBatteryIdle(connectionTest()); + + Map queued = onlyQueuedReport(); + TestUtils.validateRequiredParams(queued); + for (String param : Arrays.asList("events", "begin_session", "end_session", "session_id", "metrics", "method", "user_details")) { + Assert.assertFalse("the report carries [" + param + "]", queued.containsKey(param)); + } + + JSONObject report = new JSONObject(queued.get(keyResults)); + Assert.assertEquals(names("ts", "sdk", "results"), report.keySet()); + Assert.assertTrue(report.getLong("ts") > 0); + Assert.assertEquals(TestUtils.SDK_NAME, report.getJSONObject("sdk").getString("name")); + Assert.assertEquals(TestUtils.SDK_VERSION, report.getJSONObject("sdk").getString("version")); + assertRowOrder(report); + Map rows = rowsByFeature(report); + assertRow(rows.get("core"), true, 200, null, null); + assertRow(rows.get("core-write"), true, 400, null, null); + assertRow(rows.get("sc"), true, 200, null, null); + assertRow(rows.get("rc"), true, 400, null, null); + assertRow(rows.get("ab"), true, 400, null, null); + assertRow(rows.get("feedback"), true, 400, null, null); + assertRow(rows.get("feedback-widget"), true, 400, null, 3); + assertRow(rows.get("feedback-submit"), true, 400, null, null); + assertRow(rows.get("content"), true, 400, null, null); + assertRow(rows.get("feedback-page"), true, 200, null, 3); + assertRow(rows.get("feedback-assets"), true, 200, null, 2); + assertRow(rows.get("content-page"), true, 200, null, null); + + List probedPaths = new ArrayList<>(); + for (Received probe : server.probes) { + Assert.assertEquals("GET", probe.method); + Assert.assertTrue(probe.path, probe.path.startsWith(PREFIX + "/")); + Assert.assertFalse(probe.path, probe.path.contains("//")); + Assert.assertTrue(probe.rawQuery, probe.rawQuery.matches("(method=[a-z_]+&)?ct=1&_=\\d+")); + Assert.assertEquals(0, probe.bodyLength); + Assert.assertEquals("yes", probe.headers.get("x-countly-test")); + String method = probe.params.get("method"); + probedPaths.add(probe.path.substring(PREFIX.length()) + (method == null ? "" : "?method=" + method)); + } + Assert.assertEquals(EXPECTED_PATHS, probedPaths); + + Assert.assertEquals(names("v", "t", "c"), storedConfig().keySet()); + Assert.assertFalse(provider().getViewTrackingEnabled()); + + Assert.assertTrue("the report never reached the server", waitFor(() -> !reportsSentToTheServer().isEmpty())); + Received sent = reportsSentToTheServer().get(0); + Assert.assertEquals(queued.get(keyResults), sent.params.get(keyResults)); + Assert.assertEquals("yes", sent.headers.get("x-countly-test")); + } + + /** + * A server that fails in every way the grading tells apart. + *

+ * Verifies that {@code /o/ping} answering {@code 404 DB Error} and an image route answering 404 + * are faults on the two rows that need a 2xx, that a 403, a 5xx and a redirect are faults on any + * row, each with its own {@code e}, that a combined row keeps probing after its first failing + * path and reports that path's status, and that the redirect is read as a 3xx and never + * followed. + */ + @Test + public void armed_brokenServer_gradesEveryFailureToItsReason_andNeverFollowsARedirect() throws Exception { + server.overrides.put(PREFIX + "/o/ping", Answer.json(404, "{\"result\":\"DB Error\"}")); + server.overrides.put(PREFIX + "/i", Answer.page(403, "text/html", "forbidden")); + server.overrides.put(PREFIX + "/o/sdk", Answer.json(500, "{\"result\":\"boom\"}")); + server.overrides.put(PREFIX + "/o/surveys/survey/widget", Answer.redirect(PREFIX + "/o/ping")); + server.overrides.put(PREFIX + "/star-rating/images/ct-probe.png", Answer.page(404, "text/html", "no such image")); + server.overrides.put(PREFIX + "/feedback/rating", Answer.page(502, "text/html", "bad gateway")); + settings.respondWith(new ServerConfigBuilder().connectionTest(true)); + init(config(server.url(PREFIX))); + awaitBatteryIdle(connectionTest()); + + JSONObject report = new JSONObject(onlyQueuedReport().get(keyResults)); + assertRowOrder(report); + Map rows = rowsByFeature(report); + assertRow(rows.get("core"), false, 404, "HTTP 404", null); + assertRow(rows.get("core-write"), false, 403, "HTTP 403", null); + assertRow(rows.get("sc"), true, 200, null, null); + assertRow(rows.get("rc"), false, 500, "HTTP 500", null); + assertRow(rows.get("ab"), false, 500, "HTTP 500", null); + assertRow(rows.get("feedback"), false, 500, "HTTP 500", null); + assertRow(rows.get("feedback-widget"), false, 302, "redirected", 3); + assertRow(rows.get("feedback-submit"), true, 400, null, null); + assertRow(rows.get("content"), true, 400, null, null); + assertRow(rows.get("feedback-page"), false, 502, "HTTP 502", 3); + assertRow(rows.get("feedback-assets"), false, 404, "HTTP 404", 2); + assertRow(rows.get("content-page"), true, 200, null, null); + + Assert.assertEquals(16, server.probes.size()); + Assert.assertEquals(1, probesTo(PREFIX + "/o/ping")); + Assert.assertEquals(1, probesTo(PREFIX + "/o/feedback/widget")); + } + + /** + * A response without the flag, or with a falsy one. + *

+ * Verifies, for an absent flag and for {@code 0}, {@code false}, {@code "0"}, {@code "false"}, + * {@code ""} and {@code null}, each delivered by the fetch made at init, that no battery starts, + * no probe reaches the server and no report is queued, while the settings of the same response + * apply and the flag is never stored. + */ + @Test + public void notArmed_anAbsentOrFalsyFlag_runsNothing_whileTheSettingsStillApply() throws Exception { + Object[] variants = { null, 0, false, "0", "false", "", JSONObject.NULL }; + for (Object variant : variants) { + String label = "ct = " + (variant == null ? "absent" : JSONObject.valueToString(variant)); + TestUtils.createCleanTestState(); + ServerConfigBuilder response = new ServerConfigBuilder().viewTracking(false); + if (variant != null) { + response.connectionTest(variant); + } + settings.respondWith(response); + init(config(server.url(PREFIX))); + + Assert.assertFalse(label, provider().getViewTrackingEnabled()); + Assert.assertFalse(label, connectionTest().batteryRunning.get()); + Assert.assertEquals(label, 0, logsContaining(BATTERY_STARTED)); + Assert.assertEquals(label, names("v", "t", "c"), storedConfig().keySet()); + Assert.assertTrue(label, queuedReports().isEmpty()); + } + + Assert.assertEquals(variants.length, settings.requests.size()); + Assert.assertTrue(server.probes.isEmpty()); + } + + /** + * An armed response that also switches networking off. + *

+ * Verifies that the battery is skipped, so no probe and no report go out, and that nothing about + * the skipped test is kept: the next armed response, which allows networking again, runs the + * battery and queues its report. + */ + @Test + public void armed_whileNetworkingIsDisabled_skipsTheBattery_andTheNextArmedResponseRunsIt() throws Exception { + settings.respondWith(new ServerConfigBuilder().networking(false).connectionTest(1)); + init(config(server.url(PREFIX))); + ModuleConnectionTest module = connectionTest(); + + Assert.assertFalse(provider().getNetworkingEnabled()); + Assert.assertFalse(module.batteryRunning.get()); + Assert.assertEquals(1, logsContaining(NETWORKING_DISABLED)); + Assert.assertEquals(0, logsContaining(BATTERY_STARTED)); + Assert.assertTrue(queuedReports().isEmpty()); + Assert.assertTrue(server.probes.isEmpty()); + + push(new ServerConfigBuilder().networking(true).connectionTest(1)); + awaitBatteryIdle(module); + + Assert.assertEquals(16, server.probes.size()); + assertRowOrder(new JSONObject(onlyQueuedReport().get(keyResults))); + } + + /** + * An armed SDK whose server refuses every connection. + *

+ * Verifies that every probed row is a transport error with status 0, keeping its path count, + * that the {@code sc} row still carries the measured settings request, and that the report is + * queued with the SDK identity and stays queued after the queue failed to send it. + */ + @Test + public void armed_serverDown_reportsATransportErrorOnEveryProbedRow_andTheReportStaysQueued() throws Exception { + settings.respondWith(new ServerConfigBuilder().connectionTest("1")); + init(config(deadUrl())); + awaitBatteryIdle(connectionTest()); + + Assert.assertTrue("the queue never tried to send the report, logs: " + logs, + waitFor(() -> logs.stream().anyMatch(line -> line.contains("Error while sending request") && line.contains(keyResults)))); + Map queued = onlyQueuedReport(); + Assert.assertEquals(TestUtils.SERVER_APP_KEY, queued.get("app_key")); + Assert.assertEquals(TestUtils.DEVICE_ID, queued.get("device_id")); + + JSONObject report = new JSONObject(queued.get(keyResults)); + assertRowOrder(report); + Map rows = rowsByFeature(report); + Map pathCounts = new HashMap<>(); + pathCounts.put("feedback-widget", 3); + pathCounts.put("feedback-page", 3); + pathCounts.put("feedback-assets", 2); + for (String feature : EXPECTED_ORDER) { + if ("sc".equals(feature)) { + assertRow(rows.get(feature), true, 200, null, null); + } else { + assertRow(rows.get(feature), false, 0, "transport error", pathCounts.get(feature)); + } + } + } + + /** + * An endpoint that accepts the connection but never answers. + *

+ * Verifies with a one second per request timeout that the silent endpoint is reported as a + * timeout with status 0 after at least the timeout, while the rows around it are unaffected. + */ + @Test + public void armed_silentEndpoint_reportsATimeout_andTheOtherRowsAreUnaffected() throws Exception { + server.overrides.put(PREFIX + "/o/sdk/content", Answer.SILENT); + settings.pending(); + init(config(server.url(PREFIX))); + ModuleConnectionTest module = connectionTest(); + module.perRequestTimeoutMs = 1_000; + + push(new ServerConfigBuilder().connectionTest(1)); + awaitBatteryIdle(module); + + Map rows = rowsByFeature(new JSONObject(onlyQueuedReport().get(keyResults))); + JSONObject content = rows.get("content"); + assertRow(content, false, 0, "timeout", null); + Assert.assertTrue("ms " + content.getLong("ms"), content.getLong("ms") >= 1_000); + assertRow(rows.get("core"), true, 200, null, null); + assertRow(rows.get("feedback-submit"), true, 400, null, null); + assertRow(rows.get("feedback-page"), true, 200, null, 3); + assertRow(rows.get("content-page"), true, 200, null, null); + } + + /** + * Deliveries that overlap a running battery, and one after it. + *

+ * Verifies that a delivery while a battery runs is ignored, that the battery runs on a daemon + * thread named {@value ModuleConnectionTest#THREAD_NAME}, never on the thread that delivered the + * response, and hands every probe the transport of the queued requests, and that a delivery after + * the battery finished runs it again with the latency of its own settings request. + */ + @Test + public void aDeliveryWhileABatteryRuns_isIgnored_andOneAfterItRunsAgain_onADaemonThreadOfItsOwn() throws Exception { + settings.pending(); + init(config(server.url(PREFIX))); + ModuleConnectionTest module = connectionTest(); + FakeTransport first = FakeTransport.gated(); + module.transport = first; + + push(new ServerConfigBuilder().connectionTest(1)); + Assert.assertTrue(first.firstProbeStarted.await(WAIT_BUDGET_MS, TimeUnit.MILLISECONDS)); + Assert.assertTrue(module.batteryRunning.get()); + + push(new ServerConfigBuilder().connectionTest(true)); + push(new ServerConfigBuilder().connectionTest("yes")); + Assert.assertEquals(2, logsContaining(ALREADY_RUNNING)); + Assert.assertEquals(1, first.urls.size()); + + first.gate.countDown(); + awaitBatteryIdle(module); + Assert.assertEquals(16, first.urls.size()); + Assert.assertEquals(Collections.singleton(ModuleConnectionTest.THREAD_NAME), first.threadNames); + Assert.assertFalse(first.daemonFlags.contains(false)); + Transport queueTransport = SDKCore.instance.networking.getTransport(); + Assert.assertNotNull(queueTransport); + for (Transport used : first.transports) { + Assert.assertSame(queueTransport, used); + } + Assert.assertEquals(1, queuedReports().size()); + + FakeTransport second = FakeTransport.instant(1); + module.transport = second; + settings.delayedBy(80); + push(new ServerConfigBuilder().connectionTest(1)); + awaitBatteryIdle(module); + + Assert.assertEquals(16, second.urls.size()); + Assert.assertEquals(2, logsContaining(BATTERY_STARTED)); + List> reports = queuedReports(); + Assert.assertEquals(2, reports.size()); + long latencyMs = rowsByFeature(new JSONObject(reports.get(1).get(keyResults))).get("sc").getLong("ms"); + Assert.assertTrue("sc ms " + latencyMs, latencyMs >= 70); + } + + /** + * The SDK stops while a battery runs. + *

+ * Verifies that the module listens to the configuration module of its SDK from init and stops + * listening on stop, that the stopped battery ends before its next row and queues no report, + * not even into the queue of an SDK started again meanwhile, and that the stopped module refuses + * a later delivery. + */ + @Test + public void stop_duringABattery_endsItBeforeTheNextRow_queuesNoReport_andStopsListening() throws Exception { + settings.pending(); + init(config(server.url(PREFIX))); + ModuleConnectionTest module = connectionTest(); + ModuleConfiguration configuration = SDKCore.instance.module(ModuleConfiguration.class); + Assert.assertSame(module, SDKCore.instance.modules.get(0)); + Assert.assertSame(module, configuration.connectionTestListener); + FakeTransport gated = FakeTransport.gated(); + module.transport = gated; + + push(new ServerConfigBuilder().connectionTest(1)); + Assert.assertTrue(gated.firstProbeStarted.await(WAIT_BUDGET_MS, TimeUnit.MILLISECONDS)); + Countly.instance().stop(); + + Assert.assertTrue(module.halted); + Assert.assertNull(configuration.connectionTestListener); + + //started before the stopped battery moves on, so a report it queued would land in this queue + settings.pending(); + init(config(server.url(PREFIX))); + gated.gate.countDown(); + awaitBatteryIdle(module); + + Assert.assertEquals(1, gated.urls.size()); + Assert.assertEquals(1, logsContaining(REPORT_DROPPED)); + Assert.assertTrue(queuedReports().isEmpty()); + + module.onConnectionTestRequested(5); + Assert.assertFalse(module.batteryRunning.get()); + Assert.assertEquals(1, gated.urls.size()); + Assert.assertNotSame(module, connectionTest()); + Assert.assertSame(connectionTest(), SDKCore.instance.module(ModuleConfiguration.class).connectionTestListener); + } + + /** + * The battery cap, attempted requests times the per request timeout plus the extra time. + *

+ * Verifies with a 160 ms cap and probes that take 40 ms each that the first row is always + * probed, that the {@code sc} row is never capped, that once the cap is reached every later + * probed row is reported as not run, with status 0, 0 ms and no path count, that no probe is + * issued for those rows, and that the report still goes out. + */ + @Test + public void batteryCap_reportsEveryRowItIsReachedBeforeAsNotRun_andTheReportStillGoesOut() throws Exception { + settings.pending(); + init(config(server.url(PREFIX))); + ModuleConnectionTest module = connectionTest(); + module.perRequestTimeoutMs = 10; + module.batteryCapExtraMs = 0; + FakeTransport slow = FakeTransport.slow(40); + module.transport = slow; + + push(new ServerConfigBuilder().connectionTest(1)); + awaitBatteryIdle(module); + + JSONObject report = new JSONObject(onlyQueuedReport().get(keyResults)); + assertRowOrder(report); + JSONArray results = report.getJSONArray("results"); + assertRow(results.getJSONObject(0), true, 200, null, null); + assertRow(results.getJSONObject(2), true, 200, null, null); + + boolean capReached = false; + int probedPaths = 0; + for (int i = 0; i < results.length(); i++) { + JSONObject row = results.getJSONObject(i); + String feature = row.getString("f"); + if ("sc".equals(feature)) { + continue; + } + if (NOT_RUN.equals(row.optString("e"))) { + capReached = true; + assertRow(row, false, 0, NOT_RUN, null); + Assert.assertEquals(feature, 0, row.getLong("ms")); + } else { + Assert.assertFalse(feature + " was probed after the cap was reached", capReached); + Assert.assertTrue(feature, row.getBoolean("ok")); + probedPaths += row.optInt("n", 1); + } + } + Assert.assertTrue(capReached); + Assert.assertEquals(NOT_RUN, results.getJSONObject(results.length() - 1).getString("e")); + Assert.assertEquals(probedPaths, slow.urls.size()); + Assert.assertTrue(slow.urls.size() < 16); + } + + /** + * The grading table. + *

+ * Verifies that a 2xx is reached, that a 4xx other than 403 is reached except on a row that + * needs a 2xx, that a redirect, a 403, a 5xx and an informational status are faults with their + * {@code e}, and that no status reports its failure, a transport error when none is known. + */ + @Test + public void grade_followsTheGradingTable() { + Object[][] cases = { + // status, failure, requires a 2xx, expected e (null when reached) + { 200, null, false, null }, { 204, null, false, null }, { 400, null, false, null }, { 404, null, false, null }, + { 499, null, false, null }, { 403, null, false, "HTTP 403" }, { 500, null, false, "HTTP 500" }, { 502, null, false, "HTTP 502" }, + { 503, null, false, "HTTP 503" }, { 301, null, false, "redirected" }, { 302, null, false, "redirected" }, + { 307, null, false, "redirected" }, { 308, null, false, "redirected" }, { 100, null, false, "HTTP 100" }, + { 0, "timeout", false, "timeout" }, { 0, "transport error", false, "transport error" }, { 0, null, false, "transport error" }, + { -1, null, false, "transport error" }, + { 200, null, true, null }, { 299, null, true, null }, { 400, null, true, "HTTP 400" }, { 404, null, true, "HTTP 404" }, + { 403, null, true, "HTTP 403" }, { 302, null, true, "redirected" }, { 500, null, true, "HTTP 500" }, { 0, "timeout", true, "timeout" }, + }; + for (Object[] c : cases) { + ProbeOutcome outcome = new ProbeOutcome(); + outcome.status = (int) c[0]; + outcome.failure = (String) c[1]; + Assert.assertEquals("status " + c[0] + ", requires a 2xx " + c[2], c[3], ModuleConnectionTest.grade(outcome, (boolean) c[2])); + } + } + + /** + * Probe URLs. + *

+ * Verifies that a path resolves against the configured URL with its path prefix kept, trailing + * slashes stripped and exactly one slash at the join, that the marker and the cache buster follow + * a {@code ?}, or an {@code &} after the method selector of a path that has one, and that the + * cache buster is the current time. + */ + @Test + public void buildProbeUrl_keepsThePathPrefix_stripsTrailingSlashes_andAppendsTheMarkerAndTheCacheBuster() { + long before = System.currentTimeMillis(); + String plain = ModuleConnectionTest.buildProbeUrl("https://x.com", "/o/ping"); + long after = System.currentTimeMillis(); + Assert.assertTrue(plain, plain.startsWith("https://x.com/o/ping?ct=1&_=")); + long cacheBuster = Long.parseLong(plain.substring(plain.indexOf("_=") + 2)); + Assert.assertTrue(cacheBuster >= before && cacheBuster <= after); + + Assert.assertTrue(ModuleConnectionTest.buildProbeUrl("https://x.com/countly", "/o/ping").startsWith("https://x.com/countly/o/ping?ct=1&_=")); + Assert.assertTrue(ModuleConnectionTest.buildProbeUrl("https://x.com/countly/", "/o/ping").startsWith("https://x.com/countly/o/ping?ct=1&_=")); + Assert.assertTrue(ModuleConnectionTest.buildProbeUrl("https://x.com/countly///", "o/ping").startsWith("https://x.com/countly/o/ping?ct=1&_=")); + Assert.assertTrue(ModuleConnectionTest.buildProbeUrl("http://127.0.0.1:8080/a/b/", "/_external/content/").startsWith("http://127.0.0.1:8080/a/b/_external/content/?ct=1&_=")); + + String withSelector = ModuleConnectionTest.buildProbeUrl("https://x.com/countly/", "/o/sdk?method=rc"); + Assert.assertTrue(withSelector, withSelector.matches("https://x\\.com/countly/o/sdk\\?method=rc&ct=1&_=\\d+")); + } + + /** + * The caps of the report. + *

+ * Verifies that a report keeps at most 32 rows, that one over 8 KB loses every {@code e} except + * {@code opaque} before any row, that a small one keeps its {@code e} cut at 256 characters, and + * that rows too large even without their {@code e} are dropped from the end, the size counted in + * UTF-8 bytes rather than characters. + */ + @Test + public void buildReport_keepsAtMost32Rows_within8KB_dropsErrorsBeforeRows_andOpaqueSurvives() { + init(TestUtils.getBaseConfig()); + ModuleConnectionTest module = connectionTest(); + String longError = repeat('x', 300); + + List rows = new ArrayList<>(); + for (int i = 0; i < 40; i++) { + rows.add(resultRow("row" + i, false, 403, i, longError, 1)); + } + rows.set(5, resultRow("opaque-row", true, 0, 3, "opaque", 1)); + + JSONObject report = new JSONObject(module.buildReport(rows)); + JSONArray results = report.getJSONArray("results"); + Assert.assertEquals(32, results.length()); + Assert.assertTrue(ModuleConnectionTest.utf8Length(report.toString()) <= 8 * 1024); + for (int i = 0; i < results.length(); i++) { + JSONObject row = results.getJSONObject(i); + if (i == 5) { + Assert.assertEquals("opaque-row", row.getString("f")); + Assert.assertEquals("opaque", row.getString("e")); + } else { + Assert.assertEquals("row" + i, row.getString("f")); + Assert.assertFalse(row.has("e")); + Assert.assertEquals(403, row.getInt("st")); + Assert.assertEquals(i, row.getLong("ms")); + } + } + + JSONObject small = new JSONObject(module.buildReport(Collections.singletonList(resultRow("core", false, 500, 10, longError, 1)))); + JSONObject row = small.getJSONArray("results").getJSONObject(0); + Assert.assertEquals(longError.substring(0, 256), row.getString("e")); + Assert.assertEquals(500, row.getInt("st")); + Assert.assertFalse(row.has("n")); + Assert.assertTrue(small.getLong("ts") > 0); + Assert.assertEquals(TestUtils.SDK_NAME, small.getJSONObject("sdk").getString("name")); + + List wide = new ArrayList<>(); + JSONArray untrimmed = new JSONArray(); + for (int i = 0; i < 32; i++) { + ResultRow wideRow = resultRow(i + "-" + repeat('ü', 150), true, 200, 0, "opaque", 1); + wide.add(wideRow); + untrimmed.put(ModuleConnectionTest.rowToJson(wideRow)); + } + Assert.assertTrue("32 rows fit by characters", untrimmed.toString().length() < 8 * 1024); + JSONObject trimmed = new JSONObject(module.buildReport(wide)); + JSONArray kept = trimmed.getJSONArray("results"); + Assert.assertTrue("kept " + kept.length(), kept.length() > 0 && kept.length() < 32); + Assert.assertTrue(ModuleConnectionTest.utf8Length(trimmed.toString()) <= 8 * 1024); + for (int i = 0; i < kept.length(); i++) { + Assert.assertTrue(kept.getJSONObject(i).getString("f").startsWith(i + "-")); + Assert.assertEquals("opaque", kept.getJSONObject(i).getString("e")); + } + } + + /** + * A battery without a measured settings request. + *

+ * Verifies that the {@code sc} row is left out rather than invented, that the other rows keep + * their order, and that a combined row sums the latencies of its paths and reports how many it + * probed, which a single path row leaves out of its JSON. + */ + @Test + public void runBattery_withoutAScLatency_leavesTheScRowOut_andCombinesEveryMultiPathRow() { + init(TestUtils.getBaseConfig()); + ModuleConnectionTest module = connectionTest(); + module.transport = FakeTransport.instant(2); + + List rows = module.runBattery(-1); + + List expected = new ArrayList<>(Arrays.asList(EXPECTED_ORDER)); + expected.remove("sc"); + List features = new ArrayList<>(); + Map byFeature = new HashMap<>(); + for (ResultRow row : rows) { + features.add(row.feature); + byFeature.put(row.feature, row); + Assert.assertTrue(row.feature, row.ok); + Assert.assertNull(row.feature, row.error); + } + Assert.assertEquals(expected, features); + assertCombined(byFeature.get("feedback-widget"), 400, 6, 3); + assertCombined(byFeature.get("feedback-page"), 400, 6, 3); + assertCombined(byFeature.get("feedback-assets"), 200, 4, 2); + assertCombined(byFeature.get("core"), 200, 2, 1); + Assert.assertFalse(ModuleConnectionTest.rowToJson(byFeature.get("core")).has("n")); + Assert.assertEquals(3, ModuleConnectionTest.rowToJson(byFeature.get("feedback-widget")).getInt("n")); + } + + /** + * The connection a probe is sent on. + *

+ * Verifies that it is a bare GET that follows no redirect, without touching the default of + * every connection, uses no cache and has the given timeouts, that it carries the custom request + * headers with a usable name and value, that an HTTPS probe verifies the server against the pins + * only when pins are configured, and that the plain connection used without a transport is the + * same bare GET without any header. + */ + @Test + public void probeConnection_isABareGet_withTheCustomHeaders_andThePinsOnlyWhenPinned() throws Exception { + Map headers = new HashMap<>(); + headers.put("X-Countly-Test", "yes"); + headers.put("", "a header without a name"); + headers.put("X-Null-Value", null); + Transport unpinned = initializedTransport(TestUtils.getBaseConfig().addCustomNetworkRequestHeaders(headers)); + + HttpURLConnection probe = unpinned.openProbeConnection("http://127.0.0.1:9" + PREFIX + "/o/ping?ct=1&_=5", 1_234); + assertBareGet(probe, 1_234); + Assert.assertTrue(HttpURLConnection.getFollowRedirects()); + Assert.assertEquals("yes", probe.getRequestProperty("X-Countly-Test")); + Assert.assertEquals(1, probe.getRequestProperties().size()); + + HttpsURLConnection unpinnedHttps = (HttpsURLConnection) unpinned.openProbeConnection("https://127.0.0.1:9/o/ping", 100); + Assert.assertSame(HttpsURLConnection.getDefaultSSLSocketFactory(), unpinnedHttps.getSSLSocketFactory()); + + Transport pinned = initializedTransport(TestUtils.getBaseConfig().addPublicKeyPin(generatedPublicKeyPin())); + HttpsURLConnection pinnedHttps = (HttpsURLConnection) pinned.openProbeConnection("https://127.0.0.1:9/o/ping", 100); + assertBareGet(pinnedHttps, 100); + Assert.assertNotSame(HttpsURLConnection.getDefaultSSLSocketFactory(), pinnedHttps.getSSLSocketFactory()); + Assert.assertTrue(pinnedHttps.getRequestProperties().isEmpty()); + + HttpURLConnection bare = Transport.openBareGetConnection("http://127.0.0.1:9/o/ping", 77); + assertBareGet(bare, 77); + Assert.assertTrue(bare.getRequestProperties().isEmpty()); + } + + /** + * The default probe transport, used without a transport of the SDK. + *

+ * Verifies against the local server that it reads the real status of a 2xx, a 4xx and a 3xx, + * which it never follows, that it reads a body larger than it drains, that a silent endpoint + * becomes a timeout with status 0 after at least the timeout, and that a refused connection + * becomes a transport error with status 0. + */ + @Test + public void httpProbe_readsTheRealStatus_neverFollowsARedirect_andMapsEveryFailureToItsReason() throws Exception { + server.overrides.put(PREFIX + "/moved", Answer.redirect(PREFIX + "/o/ping")); + server.overrides.put(PREFIX + "/large", Answer.page(200, "text/html", repeat('x', 200 * 1024))); + server.overrides.put(PREFIX + "/silent", Answer.SILENT); + ModuleConnectionTest.HttpProbeTransport http = new ModuleConnectionTest.HttpProbeTransport(); + + assertOutcome(http.probe(server.url(PREFIX + "/o/ping?ct=1"), 2_000, null), 200, null); + assertOutcome(http.probe(server.url(PREFIX + "/i?ct=1"), 2_000, null), 400, null); + assertOutcome(http.probe(server.url(PREFIX + "/o/feedback/widget?ct=1"), 2_000, null), 404, null); + assertOutcome(http.probe(server.url(PREFIX + "/moved?ct=1"), 2_000, null), 302, null); + assertOutcome(http.probe(server.url(PREFIX + "/large?ct=1"), 2_000, null), 200, null); + Assert.assertEquals(1, probesTo(PREFIX + "/o/ping")); + + ProbeOutcome silent = http.probe(server.url(PREFIX + "/silent?ct=1"), 300, null); + assertOutcome(silent, 0, "timeout"); + Assert.assertTrue("ms " + silent.ms, silent.ms >= 300); + + assertOutcome(http.probe(deadUrl() + "/o/ping?ct=1", 2_000, null), 0, "transport error"); + } + + /** + * The rules of the request queue that the report is subject to. + *

+ * Verifies that the module is built and the report queued while consent is required and none is + * given, as the probes carry no identity, and that while the settings switch tracking off the + * battery still runs but the queue drops its report. + */ + @Test + public void report_isQueuedWithoutAnyConsent_butDroppedWhileTrackingIsOff() throws Exception { + settings.respondWith(new ServerConfigBuilder().connectionTest(1)); + init(config(server.url(PREFIX)).setRequiresConsent(true).enableFeatures(Config.Feature.Events)); + ModuleConnectionTest module = connectionTest(); + awaitBatteryIdle(module); + + Assert.assertTrue(SDKCore.instance.config.requiresConsent()); + Assert.assertFalse(Countly.isTracking(Config.Feature.Events)); + Assert.assertSame(module, SDKCore.instance.modules.get(0)); + Assert.assertEquals(16, server.probes.size()); + Assert.assertEquals(1, queuedReports().size()); + + push(new ServerConfigBuilder().tracking(false).connectionTest(1)); + awaitBatteryIdle(module); + + Assert.assertEquals(32, server.probes.size()); + Assert.assertEquals(1, queuedReports().size()); + Assert.assertEquals(1, logsContaining("[ModuleRequests] pushAsync, tracking disabled by SDK behavior settings; dropping request")); + } + + // endregion + // region helpers + + /** + * One request as the local server saw it. + */ + private static final class Received { + final String method; + final String path; + final String rawQuery; + final int bodyLength; + final Map headers = new HashMap<>(); + final Map params = new LinkedHashMap<>(); + + /** + * Records a request as it arrives: its method, path, query, body, headers by lower case + * name, and its parameters, decoded from the query and the body. + * + * @param exchange the request + * @throws IOException when the body cannot be read + */ + Received(HttpExchange exchange) throws IOException { + method = exchange.getRequestMethod(); + path = exchange.getRequestURI().getPath(); + rawQuery = exchange.getRequestURI().getRawQuery(); + byte[] body = readAll(exchange.getRequestBody()); + bodyLength = body.length; + for (Map.Entry> header : exchange.getRequestHeaders().entrySet()) { + if (!header.getValue().isEmpty()) { + headers.put(header.getKey().toLowerCase(Locale.ROOT), header.getValue().get(0)); + } + } + addParams(rawQuery); + addParams(new String(body, StandardCharsets.UTF_8)); + } + + /** + * Decodes url encoded parameters into {@link #params}. + * + * @param encoded the parameters, {@code null} for none + */ + private void addParams(String encoded) { + if (encoded == null || encoded.isEmpty()) { + return; + } + for (String pair : encoded.split("&")) { + String[] keyValue = pair.split("=", 2); + params.put(Utils.urldecode(keyValue[0]), keyValue.length > 1 ? Utils.urldecode(keyValue[1]) : ""); + } + } + } + + /** + * How the local server answers a request. + */ + private static final class Answer { + static final Answer SILENT = new Answer(0, null, "", null, true); + + final int status; + final String contentType; + final String body; + final String location; + final boolean silent; + + /** + * Describes an answer. + * + * @param status the status + * @param contentType the content type + * @param body the body, empty for none + * @param location the redirect target, {@code null} for none + * @param silent whether the server holds the request without ever answering it + */ + private Answer(int status, String contentType, String body, String location, boolean silent) { + this.status = status; + this.contentType = contentType; + this.body = body; + this.location = location; + this.silent = silent; + } + + /** + * A JSON answer. + * + * @param status the status + * @param body the JSON text + * @return the answer + */ + static Answer json(int status, String body) { + return new Answer(status, "application/json", body, null, false); + } + + /** + * A page, an image or any other body. + * + * @param status the status + * @param contentType the content type + * @param body the body + * @return the answer + */ + static Answer page(int status, String contentType, String body) { + return new Answer(status, contentType, body, null, false); + } + + /** + * A 302 to the given location, without a body. + * + * @param location the redirect target + * @return the answer + */ + static Answer redirect(String location) { + return new Answer(302, "text/html", "", location, false); + } + } + + /** + * A local stand in for a Countly server behind {@link #PREFIX}. A probe gets the answer a + * healthy server gives a request without app key and device ID, unless a test overrides its + * path: {@code /o/ping} 200, {@code /o/feedback/widget} 404, the frontend pages 200 or 400, the + * images 200 and every other API path 400. The SDK's own requests, which carry the app key, are + * recorded and refused, so they stay queued. Handlers run on a thread pool, so a silent answer + * never holds up the next probe. + */ + private static final class ProbeServer { + final List probes = new CopyOnWriteArrayList<>(); + final List sdkRequests = new CopyOnWriteArrayList<>(); + final Map overrides = new ConcurrentHashMap<>(); + private final CountDownLatch silenceEnds = new CountDownLatch(1); + private final ExecutorService executor = Executors.newCachedThreadPool(runnable -> { + Thread thread = new Thread(runnable, "ct-probe-server"); + thread.setDaemon(true); + return thread; + }); + private final HttpServer http; + + /** + * Starts the server on a free loopback port. + * + * @throws IOException when the server cannot start + */ + ProbeServer() throws IOException { + http = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + http.createContext("/", this::handle); + http.setExecutor(executor); + http.start(); + } + + /** + * The URL of a path on this server. + * + * @param path the path, with any query + * @return the URL + */ + String url(String path) { + return "http://127.0.0.1:" + http.getAddress().getPort() + path; + } + + /** + * Records a request and answers it. + * + * @param exchange the request and its response + * @throws IOException when the answer cannot be written + */ + private void handle(HttpExchange exchange) throws IOException { + Received received = new Received(exchange); + if (received.params.containsKey("app_key")) { + sdkRequests.add(received); + send(exchange, Answer.json(500, "{\"result\":\"refused by the test server\"}")); + return; + } + + probes.add(received); + Answer override = overrides.get(received.path); + send(exchange, override != null ? override : healthyAnswer(received.path)); + } + + /** + * What a healthy server answers a bare request to a path. + * + * @param path the request path + * @return the answer + */ + private static Answer healthyAnswer(String path) { + if (!path.startsWith(PREFIX + "/")) { + return Answer.page(404, "text/html", "no such location"); + } + + switch (path.substring(PREFIX.length())) { + case "/o/ping": + return Answer.json(200, "{\"result\":\"Success\"}"); + case "/o/feedback/widget": + return Answer.json(404, "{\"result\":\"Widget not found\"}"); + case "/feedback/nps": + case "/feedback/survey": + case "/_external/content/": + return Answer.page(200, "text/html", ""); + case "/feedback/rating": + return Answer.page(400, "text/html", "missing widget id"); + case "/surveys/images/ct-probe.png": + case "/star-rating/images/ct-probe.png": + return Answer.page(200, "image/png", "png"); + default: + return Answer.json(400, MISSING_PARAMETER); + } + } + + /** + * Writes an answer, or holds the request until the server stops for a silent one. + * + * @param exchange the request and its response + * @param answer the answer + * @throws IOException when the answer cannot be written + */ + private void send(HttpExchange exchange, Answer answer) throws IOException { + if (answer.silent) { + try { + silenceEnds.await(WAIT_BUDGET_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + exchange.close(); + return; + } + + byte[] body = answer.body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", answer.contentType); + if (answer.location != null) { + exchange.getResponseHeaders().set("Location", answer.location); + } + exchange.sendResponseHeaders(answer.status, body.length == 0 ? -1 : body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + } + + /** + * Lets every silent request go, then stops the server and its handler threads. + */ + void stop() { + silenceEnds.countDown(); + http.stop(0); + executor.shutdownNow(); + } + } + + /** + * A probe transport without the network that answers like a healthy server, 200 for + * {@code /o/ping} and the images and 400 for everything else. It records every probe with the + * thread it ran on and the transport it was handed, and can hold every probe at a gate or take + * a given time. + */ + private static final class FakeTransport implements ModuleConnectionTest.ProbeTransport { + final List urls = new CopyOnWriteArrayList<>(); + final List transports = new CopyOnWriteArrayList<>(); + final Set threadNames = ConcurrentHashMap.newKeySet(); + final List daemonFlags = new CopyOnWriteArrayList<>(); + final CountDownLatch firstProbeStarted = new CountDownLatch(1); + final CountDownLatch gate; + private final long takesMs; + private final long reportedMs; + + /** + * Describes the fake. + * + * @param gated whether every probe waits at {@link #gate} until a test opens it + * @param takesMs how long every probe takes, in milliseconds + * @param reportedMs the latency every probe reports, in milliseconds + */ + private FakeTransport(boolean gated, long takesMs, long reportedMs) { + this.gate = new CountDownLatch(gated ? 1 : 0); + this.takesMs = takesMs; + this.reportedMs = reportedMs; + } + + /** + * A fake whose probes answer at once. + * + * @param reportedMs the latency every probe reports, in milliseconds + * @return the fake + */ + static FakeTransport instant(long reportedMs) { + return new FakeTransport(false, 0, reportedMs); + } + + /** + * A fake whose probes wait at the gate. + * + * @return the fake + */ + static FakeTransport gated() { + return new FakeTransport(true, 0, 1); + } + + /** + * A fake whose probes take the given time. + * + * @param takesMs how long every probe takes, in milliseconds + * @return the fake + */ + static FakeTransport slow(long takesMs) { + return new FakeTransport(false, takesMs, takesMs); + } + + /** + * Records the probe, waits at the gate and for the time a probe takes, then answers. + * + * @param url the probe URL + * @param timeoutMs the per request timeout + * @param transport the transport of the queued requests + * @return the answer of a healthy server + */ + @Nonnull + @Override + public ProbeOutcome probe(@Nonnull String url, int timeoutMs, @Nullable Transport transport) { + urls.add(url); + transports.add(transport); + threadNames.add(Thread.currentThread().getName()); + daemonFlags.add(Thread.currentThread().isDaemon()); + firstProbeStarted.countDown(); + try { + gate.await(WAIT_BUDGET_MS, TimeUnit.MILLISECONDS); + if (takesMs > 0) { + Thread.sleep(takesMs); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + ProbeOutcome outcome = new ProbeOutcome(); + outcome.status = url.contains("/o/ping?") || url.contains(".png?") ? 200 : 400; + outcome.ms = reportedMs; + return outcome; + } + } + + /** + * A configuration pointing at the given server URL, with no request cooldowns and every log line + * kept in {@link #logs}. + * + * @param serverUrl the server URL + * @return the configuration + */ + private Config config(String serverUrl) { + File directory = TestUtils.getTestSDirectory(); + TestUtils.checkSdkStorageRootDirectoryExist(directory); + return new Config(serverUrl, TestUtils.SERVER_APP_KEY, directory) + .setApplicationVersion(TestUtils.APPLICATION_VERSION) + .setCustomDeviceId(TestUtils.DEVICE_ID) + .setNetworkRequestCooldown(0) + .setNetworkImportantRequestCooldown(0) + .setLogListener((message, level) -> logs.add(message)); + } + + /** + * Initializes the SDK with {@link #settings} answering its settings requests. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = settings; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. The response is applied, and the test armed, before this returns. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + settings.respondWith(builder); + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * The connection test module of the running SDK. + * + * @return the module + */ + private static ModuleConnectionTest connectionTest() { + return SDKCore.instance.module(ModuleConnectionTest.class); + } + + /** + * The settings in effect in the running SDK. + * + * @return the provider + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * The settings as stored on disk. + * + * @return the stored settings, {@code null} when nothing is stored + */ + private static JSONObject storedConfig() { + String stored = TestUtils.readJsonFile(SDKStorage.JSON_FILE_NAME).optString("sc", null); + return stored == null ? null : new JSONObject(stored); + } + + /** + * Waits until no battery runs. + * + * @param module the connection test module + * @throws InterruptedException when the wait is interrupted + */ + private static void awaitBatteryIdle(ModuleConnectionTest module) throws InterruptedException { + Assert.assertTrue("the battery did not finish in time", waitFor(() -> !module.batteryRunning.get())); + } + + /** + * Every queued request that carries a report. + * + * @return the parameters of each, in queue order + */ + private static List> queuedReports() { + List> reports = new ArrayList<>(); + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey(keyResults)) { + reports.add(request); + } + } + return reports; + } + + /** + * The one queued request that carries a report. + * + * @return its parameters + */ + private static Map onlyQueuedReport() { + List> reports = queuedReports(); + Assert.assertEquals("queued reports", 1, reports.size()); + return reports.get(0); + } + + /** + * The requests carrying a report that the queue sent to the local server. + * + * @return the requests, in arrival order + */ + private List reportsSentToTheServer() { + List sent = new ArrayList<>(); + for (Received request : server.sdkRequests) { + if (request.params.containsKey(keyResults)) { + sent.add(request); + } + } + return sent; + } + + /** + * How many probes reached a path. + * + * @param path the path + * @return the number of probes + */ + private int probesTo(String path) { + int count = 0; + for (Received probe : server.probes) { + if (path.equals(probe.path)) { + count++; + } + } + return count; + } + + /** + * How many log lines contain a text. + * + * @param text the text + * @return the number of lines + */ + private int logsContaining(String text) { + int count = 0; + for (String line : logs) { + if (line.contains(text)) { + count++; + } + } + return count; + } + + /** + * The rows of a report by feature key. + * + * @param report the report + * @return the rows + */ + private static Map rowsByFeature(JSONObject report) { + JSONArray results = report.getJSONArray("results"); + Map rows = new HashMap<>(); + for (int i = 0; i < results.length(); i++) { + JSONObject row = results.getJSONObject(i); + rows.put(row.getString("f"), row); + } + return rows; + } + + /** + * Asserts that a report holds the twelve rows in table order. + * + * @param report the report + */ + private static void assertRowOrder(JSONObject report) { + JSONArray results = report.getJSONArray("results"); + Assert.assertEquals(EXPECTED_ORDER.length, results.length()); + for (int i = 0; i < EXPECTED_ORDER.length; i++) { + Assert.assertEquals(EXPECTED_ORDER[i], results.getJSONObject(i).getString("f")); + } + } + + /** + * Asserts one report row, which carries only the report keys and never {@code sk}. + * + * @param row the row + * @param ok the expected verdict + * @param st the expected status + * @param e the expected reason, {@code null} when the row has none + * @param n the expected path count, {@code null} when the row has none + */ + private static void assertRow(JSONObject row, boolean ok, int st, String e, Integer n) { + Assert.assertNotNull("missing row", row); + String label = row.getString("f"); + Assert.assertTrue(label + " keys " + row.keySet(), ROW_KEYS.containsAll(row.keySet())); + Assert.assertEquals(label + " ok", ok, row.getBoolean("ok")); + Assert.assertEquals(label + " st", st, row.getInt("st")); + Assert.assertTrue(label + " ms", row.getLong("ms") >= 0); + if (e == null) { + Assert.assertFalse(label + " has e", row.has("e")); + } else { + Assert.assertEquals(label + " e", e, row.getString("e")); + } + if (n == null) { + Assert.assertFalse(label + " has n", row.has("n")); + } else { + Assert.assertEquals(label + " n", (int) n, row.getInt("n")); + } + } + + /** + * Asserts a graded row that reached every path. + * + * @param row the row + * @param status the expected status + * @param ms the expected summed latency + * @param pathCount the expected number of paths + */ + private static void assertCombined(ResultRow row, int status, long ms, int pathCount) { + Assert.assertTrue(row.feature, row.ok); + Assert.assertEquals(row.feature, status, row.status); + Assert.assertEquals(row.feature, ms, row.ms); + Assert.assertEquals(row.feature, pathCount, row.pathCount); + } + + /** + * Asserts what a probe observed. + * + * @param outcome the outcome + * @param status the expected status + * @param failure the expected failure, {@code null} for none + */ + private static void assertOutcome(ProbeOutcome outcome, int status, String failure) { + Assert.assertEquals(status, outcome.status); + Assert.assertEquals(failure, outcome.failure); + Assert.assertTrue(outcome.ms >= 0); + } + + /** + * Asserts that a connection is a bare GET with the given timeouts. + * + * @param connection the connection, not connected yet + * @param timeoutMs the expected connect and read timeout + */ + private static void assertBareGet(HttpURLConnection connection, int timeoutMs) { + Assert.assertEquals("GET", connection.getRequestMethod()); + Assert.assertFalse(connection.getInstanceFollowRedirects()); + Assert.assertFalse(connection.getUseCaches()); + Assert.assertTrue(connection.getDoInput()); + Assert.assertFalse(connection.getDoOutput()); + Assert.assertEquals(timeoutMs, connection.getConnectTimeout()); + Assert.assertEquals(timeoutMs, connection.getReadTimeout()); + } + + /** + * A graded row. + * + * @param feature the feature key + * @param ok the verdict + * @param status the status + * @param ms the latency + * @param error the reason, {@code null} for none + * @param pathCount the number of paths + * @return the row + */ + private static ResultRow resultRow(String feature, boolean ok, int status, long ms, String error, int pathCount) { + ResultRow row = new ResultRow(feature); + row.ok = ok; + row.status = status; + row.ms = ms; + row.error = error; + row.pathCount = pathCount; + return row; + } + + /** + * A transport initialized with the given configuration. + * + * @param config the configuration + * @return the transport + */ + private static Transport initializedTransport(Config config) { + Transport transport = new Transport(); + transport.init(TestUtils.getInternalConfigWithLogger(config)); + return transport; + } + + /** + * A public key pin for a key generated for the test, base64 encoded as a pin may be given. + * + * @return the pin + * @throws Exception when no key can be generated + */ + private static String generatedPublicKeyPin() throws Exception { + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA"); + generator.initialize(2048); + return Base64.getEncoder().encodeToString(generator.generateKeyPair().getPublic().getEncoded()); + } + + /** + * The URL of a loopback port nothing listens on. + * + * @return the URL, with {@link #PREFIX} + * @throws IOException when no free port can be found + */ + private static String deadUrl() throws IOException { + try (ServerSocket socket = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + return "http://127.0.0.1:" + socket.getLocalPort() + PREFIX; + } + } + + /** + * A text of one character repeated. + * + * @param character the character + * @param count how many times + * @return the text + */ + private static String repeat(char character, int count) { + char[] characters = new char[count]; + Arrays.fill(characters, character); + return new String(characters); + } + + /** + * Reads a stream to its end. + * + * @param stream the stream + * @return the bytes + * @throws IOException when the stream cannot be read + */ + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = stream.read(buffer)) != -1) { + bytes.write(buffer, 0, read); + } + return bytes.toByteArray(); + } + + /** + * Polls a condition until it holds or the wait budget runs out. + * + * @param condition the condition + * @return whether it held + * @throws InterruptedException when the wait is interrupted + */ + private static boolean waitFor(BooleanSupplier condition) throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_BUDGET_MS; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return true; + } + Thread.sleep(POLL_MS); + } + return condition.getAsBoolean(); + } + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleContentTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleContentTests.java index 1691226a..81639399 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleContentTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleContentTests.java @@ -129,8 +129,9 @@ public void enterContentZone_withoutADisplay_isIgnored() throws JSONException { } /** - * With consent required, the content interface is unreachable until content consent is given, - * and a zone that is already running is torn down when that consent is taken away again. + * With consent required, the content interface ignores calls until content consent is given, and + * a zone that is already running is torn down when that consent is taken away again, after which + * the interface kept from before ignores calls too. */ @Test public void content_isGatedByConsent() throws JSONException { @@ -138,7 +139,7 @@ public void content_isGatedByConsent() throws JSONException { init(config); nextResponse = new JSONObject(CONTENT_RESPONSE); - Assert.assertNull(Countly.instance().content()); + Assert.assertSame(SDKCore.dormantContent, Countly.instance().content()); Countly.onConsent(Config.Feature.Content, Config.Feature.Events); installRequestMaker(); @@ -151,7 +152,9 @@ public void content_isGatedByConsent() throws JSONException { Assert.assertEquals(1, requests.size()); Countly.onConsentRemoval(Config.Feature.Content); - Assert.assertNull(Countly.instance().content()); + Assert.assertSame(SDKCore.dormantContent, Countly.instance().content()); + content.enterContentZone(); + content.refreshContentZone(); Assert.assertEquals(1, requests.size()); } diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleCrashesTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleCrashesTests.java index 84d3f234..3ea89209 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleCrashesTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleCrashesTests.java @@ -2,8 +2,12 @@ import java.io.PrintWriter; import java.io.StringWriter; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import ly.count.sdk.java.Config; import ly.count.sdk.java.Countly; @@ -146,6 +150,45 @@ public void addCrashBreadcrumb() { validateCrashInRQ(testThrowable, false, null, null, 1); } + /** + * "addCrashBreadcrumb" while the uncaught exception handler records crashes + * Validating that breadcrumbs added through the interface while crashes are recorded on another + * thread, as the uncaught exception handler does, are each taken by one crash or kept for the + * next, none of them lost, and that no call throws + */ + @Test + public void addCrashBreadcrumb_whileAnotherThreadRecordsCrashes_losesNone() throws InterruptedException { + BreadcrumbCountingProcessor.taken.set(0); + int breadcrumbs = 20_000; + Countly.instance().init(TestUtils.getBaseConfig().enableFeatures(Config.Feature.CrashReporting) + .setMaxBreadcrumbCount(breadcrumbs).setCrashProcessorClass(BreadcrumbCountingProcessor.class)); + ModuleCrashes module = SDKCore.instance.module(ModuleCrashes.class); + List failures = new CopyOnWriteArrayList<>(); + AtomicBoolean adding = new AtomicBoolean(true); + + Thread crashes = new Thread(() -> { + try { + while (adding.get()) { + module.recordExceptionInternal(new IllegalStateException("crash"), false, null, null); + } + } catch (Throwable t) { + failures.add(t); + } + }); + crashes.start(); + try { + for (int i = 0; i < breadcrumbs; i++) { + Countly.instance().crashes().addCrashBreadcrumb("breadcrumb" + i); + } + } finally { + adding.set(false); + crashes.join(30_000); + } + + Assert.assertTrue("failed: " + failures, failures.isEmpty()); + Assert.assertEquals(breadcrumbs, BreadcrumbCountingProcessor.taken.get() + module.logs.size()); + } + /** * "addCrashBreadcrumb" with default "maxBreadcrumbCount" * Validating that handled exception is recorded correctly to request queue and has breadcrumbs, @@ -270,6 +313,22 @@ private void recordExceptionWithSegment_base(boolean fatal, BiConsumer logs = crash.getLogs(); + if (logs != null) { + taken.addAndGet(logs.size()); + } + return null; + } + } + public static class TestCrashProcessor implements CrashProcessor { public TestCrashProcessor() { diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleLocationsTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleLocationsTests.java index 6784b085..139c5671 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleLocationsTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleLocationsTests.java @@ -35,6 +35,21 @@ public void disableLocation() { validateLocationRequestInRQ(TestUtils.map("location", ""), 0); } + /** + * Removing the location consent erases the location on the server right away + * Request queue should hold the location request and then one with an empty location + */ + @Test + public void locationConsentRemoval_erasesTheLocation() { + Countly.instance().init(TestUtils.getBaseConfig().enableFeatures(Config.Feature.Location).setRequiresConsent(true)); + Countly.onConsent(Config.Feature.Location); + Countly.instance().location().setLocation("TR", "Izmir", "38.4,27.1", null); + + Countly.onConsentRemoval(Config.Feature.Location); + + validateLocationRequestInRQ(TestUtils.map("location", ""), 1); + } + /** * "disableLocation" * Validating that empty location parameter is not sent because location is not enabled @@ -43,7 +58,8 @@ public void disableLocation() { @Test public void disableLocation_noConsent() { Countly.instance().init(TestUtils.getBaseConfig()); - Assert.assertNull(Countly.instance().location()); + Assert.assertSame(SDKCore.dormantLocation, Countly.instance().location()); + Countly.instance().location().disableLocation(); Assert.assertEquals(0, TestUtils.getCurrentRQ().length); } diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleRemoteConfigTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleRemoteConfigTests.java index 346a8038..0f24654d 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleRemoteConfigTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleRemoteConfigTests.java @@ -1,7 +1,11 @@ package ly.count.sdk.java.internal; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; +import ly.count.sdk.java.Config; import ly.count.sdk.java.Countly; import org.json.JSONArray; import org.json.JSONObject; @@ -134,11 +138,10 @@ public void getValue_nullKey() { @Test public void registerDownloadCallback_null() { Countly.instance().init(TestUtils.getConfigRemoteConfigs()); - ModuleRemoteConfig rcModule = SDKCore.instance.module(ModuleRemoteConfig.class); - Assert.assertEquals(0, rcModule.downloadCallbacks.size()); + Assert.assertEquals(0, SDKCore.instance.remoteConfigDownloadCallbacks.size()); Countly.instance().remoteConfig().registerDownloadCallback(null); - Assert.assertEquals(0, rcModule.downloadCallbacks.size()); + Assert.assertEquals(0, SDKCore.instance.remoteConfigDownloadCallbacks.size()); } /** @@ -149,12 +152,32 @@ public void registerDownloadCallback_null() { @Test public void registerDownloadCallback() { Countly.instance().init(TestUtils.getConfigRemoteConfigs()); - ModuleRemoteConfig rcModule = SDKCore.instance.module(ModuleRemoteConfig.class); - Assert.assertEquals(0, rcModule.downloadCallbacks.size()); + Assert.assertEquals(0, SDKCore.instance.remoteConfigDownloadCallbacks.size()); Countly.instance().remoteConfig().registerDownloadCallback((rResult, error, fullValueUpdate, downloadedValues) -> { }); - Assert.assertEquals(1, rcModule.downloadCallbacks.size()); + Assert.assertEquals(1, SDKCore.instance.remoteConfigDownloadCallbacks.size()); + } + + /** + * "registerDownloadCallback" before the consent + * Validating that a callback registered while remote config does not run is called by the module + * started once the consent is given, and that one removed meanwhile is not + */ + @Test + public void registerDownloadCallback_beforeTheConsent_isCalledOnceRemoteConfigRuns() { + Countly.instance().init(TestUtils.getConfigRemoteConfigs().setRequiresConsent(true)); + List results = new ArrayList<>(); + RCDownloadCallback removed = (rResult, error, fullValueUpdate, downloadedValues) -> results.add(null); + Countly.instance().remoteConfig().registerDownloadCallback((rResult, error, fullValueUpdate, downloadedValues) -> results.add(rResult)); + Countly.instance().remoteConfig().registerDownloadCallback(removed); + Countly.instance().remoteConfig().removeDownloadCallback(removed); + + Countly.onConsent(Config.Feature.RemoteConfig); + SDKCore.instance.config.immediateRequestGenerator = () -> remoteConfigRequestMaker(new JSONObject(), null, null, null); + Countly.instance().remoteConfig().downloadAllKeys(null); + + Assert.assertEquals(Collections.singletonList(RequestResult.Success), results); } /** @@ -165,11 +188,10 @@ public void registerDownloadCallback() { @Test public void removeDownloadCallback_null() { Countly.instance().init(TestUtils.getConfigRemoteConfigs()); - ModuleRemoteConfig rcModule = SDKCore.instance.module(ModuleRemoteConfig.class); - Assert.assertEquals(0, rcModule.downloadCallbacks.size()); + Assert.assertEquals(0, SDKCore.instance.remoteConfigDownloadCallbacks.size()); Countly.instance().remoteConfig().removeDownloadCallback(null); - Assert.assertEquals(0, rcModule.downloadCallbacks.size()); + Assert.assertEquals(0, SDKCore.instance.remoteConfigDownloadCallbacks.size()); } /** @@ -181,11 +203,10 @@ public void removeDownloadCallback_null() { public void removeDownloadCallback() { RCDownloadCallback callback = (rResult, error, fullValueUpdate, downloadedValues) -> Assert.assertEquals(RequestResult.Success, rResult); Countly.instance().init(TestUtils.getConfigRemoteConfigs().remoteConfigRegisterGlobalCallback(callback)); - ModuleRemoteConfig rcModule = SDKCore.instance.module(ModuleRemoteConfig.class); - Assert.assertEquals(1, rcModule.downloadCallbacks.size()); + Assert.assertEquals(1, SDKCore.instance.remoteConfigDownloadCallbacks.size()); Countly.instance().remoteConfig().removeDownloadCallback(callback); - Assert.assertEquals(0, rcModule.downloadCallbacks.size()); + Assert.assertEquals(0, SDKCore.instance.remoteConfigDownloadCallbacks.size()); } /** diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioInitRecoveryTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioInitRecoveryTests.java index 36ce4b34..7eb14a1f 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioInitRecoveryTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioInitRecoveryTests.java @@ -197,6 +197,20 @@ public void init_withCrashAndSessionFiles() throws InterruptedException { // ── Corrupt/empty file resilience ─────────────────────────────────── + /** + * "init_withBrokenStoreFile" + * A store file cut short, as a process stopped while writing leaves it, must not stop the SDK + * from initializing: the storage starts empty instead of throwing to the application. + */ + @Test + public void init_withBrokenStoreFile() { + TestUtils.writeToFile(SDKStorage.JSON_FILE_NAME, "{\"did\":\"generated_uuid\",\"sc\":\"{\\\"c\\\":{"); + + Countly.instance().init(TestUtils.getBaseConfig()); + + Assert.assertTrue("SDK should be initialized even with a broken store file", Countly.isInitialized()); + } + /** * "init_withCorruptCrashFile" * Corrupt crash file should not block initialization. diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioRequestQueueStallTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioRequestQueueStallTests.java index 7dab91b7..f03adca9 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioRequestQueueStallTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ScenarioRequestQueueStallTests.java @@ -94,7 +94,9 @@ private Config configForLocalServer() { .setLoggingLevel(Config.LoggingLevel.VERBOSE) .setDeviceIdStrategy(Config.DeviceIdStrategy.UUID) .enableFeatures(Config.Feature.Events, Config.Feature.Sessions) - .setEventQueueSizeToSend(1); + .setEventQueueSizeToSend(1) + // request #1 can be held past the backoff's accepted timeout, and a backoff would hold the backlog past the drain timeout + .disableBackoffMechanism(); } /** diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/SdkLogGatheringTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/SdkLogGatheringTests.java new file mode 100644 index 00000000..94a7db3d --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/SdkLogGatheringTests.java @@ -0,0 +1,1354 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import ly.count.sdk.java.internal.ConfigurationProvider.LogGatheringState; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static ly.count.sdk.java.internal.ServerConfigBuilder.names; +import static org.mockito.Mockito.mock; + +/** + * Operator armed SDK log gathering: the {@code lg} directive of a live SDK behavior settings + * response arms it, and the SDK uploads its own log lines as the {@code sdk_logs} parameter of + * requests of their own, {@code {"i":gatherId,"d":dropped,"l":[{"t":ms,"l":"e","m":message}]}}. + *

+ * The settings request goes through a {@link ModuleConfigurationTests.ServerConfigResponder}, which + * answers on the calling thread, so the directive of the fetch made at init is applied before init + * returns. Batches are queued on the delivery thread of the logger; a test waits for it with + * {@link #awaitLogDelivery()} and reads the batches off the request queue. Unless a test is about + * sending, the responses switch networking off, so every queued request stays on disk. The lines a + * test counts are logged at a level the SDK does not use in these flows, mostly {@code e}. + */ +@RunWith(JUnit4.class) +public class SdkLogGatheringTests { + + private static final String REQUEST_FILE_PREFIX = "[CLY]_request_"; + + private ModuleConfigurationTests.ServerConfigResponder server; + + /** + * Starts every test from empty storage, with the default timer and a fresh responder. + */ + @Before + public void beforeTest() { + TestUtils.createCleanTestState(); + SDKCore.testDummyModule = null; + CountlyTimer.TIMER_DELAY_MS = 0; + server = new ModuleConfigurationTests.ServerConfigResponder(); + } + + /** + * Stops the SDK, clearing its data, and restores the default timer. + */ + @After + public void afterTest() { + Countly.instance().halt(); + CountlyTimer.TIMER_DELAY_MS = 0; + } + + // region scenarios + + /** + * An armed directive adopts what was captured before it arrived. + *

+ * Verifies against a local server that the lines of the directive's levels logged during init, + * before any response, are uploaded once init finishes, the very first line of the SDK first; + * that every batch echoes the gather id, carries the required request parameters and at most + * {@code b} lines, each with exactly {@code t}, {@code l} and {@code m}, none of another level + * and none quoting a batch; that a partial batch waits for a flush while no batch is in flight; + * that a full batch goes out on its own, the lines after it following once it is sent; and that + * a full batch is too long for a GET, so it is sent as a POST. An unknown top level key next to + * {@code lg} changes none of it. + */ + @Test + public void armedDirective_uploadsTheInitLinesOfItsLevels_withTheGatherIdInBatchesOfAtMostB() throws Exception { + long startMs = System.currentTimeMillis(); + List requests = new CopyOnWriteArrayList<>(); + HttpServer httpServer = startRecordingServer(requests); + try { + server.respondWith(new ServerConfigBuilder() + .logGatheringOn("gather_init", "wi", 10) + .topLevelKey("someFutureFeature", "a value this SDK has never heard of")); + init(localConfig(httpServer)); + assertGathering("gather_init", "wi", 10); + + awaitSentBatches(); + List initBatches = receivedBatches(requests); + Assert.assertFalse("init end queued nothing", initBatches.isEmpty()); + Assert.assertEquals("[SDKCore] Initializing Countly", initBatches.get(0).getJSONArray("l").getJSONObject(0).getString("m")); + + List partialLines = numbered("[Test] partial line ", 0, 5); + for (String line : partialLines) { + logger().w(line); + } + awaitLogDelivery(); + Thread.sleep(200); + Assert.assertTrue(linesStartingWith("[Test] partial line ", receivedBatches(requests)).isEmpty()); + logger().flushGatheredLogs(); + awaitSentBatches(); + Assert.assertEquals(partialLines, linesStartingWith("[Test] partial line ", receivedBatches(requests))); + + String padding = repeat('p', 80); + List armedLines = new ArrayList<>(); + for (int i = 0; i < 25; i++) { + armedLines.add("[Test] armed line " + i + " " + padding); + logger().w(armedLines.get(i)); + } + awaitSentBatches(); + Assert.assertEquals(armedLines, linesStartingWith("[Test] armed line ", receivedBatches(requests))); + + int fullArmedBatches = 0; + for (ReceivedRequest request : requests) { + if (!request.params.containsKey(Log.keySdkLogs)) { + continue; + } + TestUtils.validateRequiredParams(request.params); + JSONObject batch = new JSONObject(request.params.get(Log.keySdkLogs)); + Assert.assertEquals(names("i", "d", "l"), batch.keySet()); + Assert.assertEquals("gather_init", batch.getString("i")); + Assert.assertEquals(0, batch.getInt("d")); + + JSONArray lines = batch.getJSONArray("l"); + Assert.assertTrue("a batch of [" + lines.length() + "] lines", lines.length() >= 1 && lines.length() <= 10); + for (int i = 0; i < lines.length(); i++) { + JSONObject line = lines.getJSONObject(i); + Assert.assertEquals(names("t", "l", "m"), line.keySet()); + long timestamp = line.getLong("t"); + Assert.assertTrue(timestamp >= startMs && timestamp <= System.currentTimeMillis()); + Assert.assertTrue("w".equals(line.getString("l")) || "i".equals(line.getString("l"))); + Assert.assertFalse(line.getString("m").contains(Log.keySdkLogs)); + } + if (request.params.get(Log.keySdkLogs).contains("[Test] armed line 9 ")) { + Assert.assertEquals("a full batch fits a GET", "POST", request.method); + fullArmedBatches++; + } + } + Assert.assertEquals(1, fullArmedBatches); + } finally { + Countly.instance().halt(); + httpServer.stop(0); + } + } + + /** + * While tracking is off the lines are held, the buffer keeps the newest 500 of them, and the + * lines it drops are reported once, with the next batch. + *

+ * Verifies against a local server that nothing is queued while tracking is off, flush or not; + * that once tracking is back on the held lines go out oldest first in full batches of {@code b}, + * one at a time as each is sent, the first one reporting every dropped line in {@code d}, so the + * uploaded and the dropped lines add up to every line logged; and that a {@code b} over the + * buffer ceiling is clamped to it, so no batch ever holds more than 500 lines. + */ + @Test + public void trackingOffHold_keepsTheNewest500Lines_andDAccountsForEveryDroppedLine() throws Exception { + List requests = new CopyOnWriteArrayList<>(); + HttpServer httpServer = startRecordingServer(requests); + try { + server.respondWith(heldGather("gather_hold", "e", 25)); + init(localConfig(httpServer)); + assertGathering("gather_hold", "e", 25); + + Log logger = logger(); + //what the SDK logged at the same level during init is held too, and is the oldest + int initLines = logger.heldLogLineCount(); + for (int i = 0; i < 520; i++) { + logger.e("[Test] held line " + i); + } + Assert.assertEquals(ModuleConfiguration.logGatheringMaxBufferedLines, logger.heldLogLineCount()); + Assert.assertEquals(initLines + 20, logger.droppedLogLineCount); + logger.flushGatheredLogs(); + awaitLogDelivery(); + Assert.assertEquals(ModuleConfiguration.logGatheringMaxBufferedLines, logger.heldLogLineCount()); + Assert.assertTrue(logRequests().isEmpty()); + + push(new ServerConfigBuilder().tracking(true).logGatheringOn("gather_hold", "e", 25)); + assertGathering("gather_hold", "e", 25); + logger.flushGatheredLogs(); + awaitSentBatches(); + Assert.assertEquals(0, logger.heldLogLineCount()); + + List batches = receivedBatches(requests); + Assert.assertEquals(20, batches.size()); + List uploaded = new ArrayList<>(); + int dropped = 0; + for (int b = 0; b < batches.size(); b++) { + JSONObject batch = batches.get(b); + Assert.assertEquals("gather_hold", batch.getString("i")); + Assert.assertEquals(b == 0 ? initLines + 20 : 0, batch.getInt("d")); + Assert.assertEquals(25, batch.getJSONArray("l").length()); + dropped += batch.getInt("d"); + uploaded.addAll(messagesOf(batch)); + } + Assert.assertEquals(numbered("[Test] held line ", 20, 520), uploaded); + Assert.assertEquals(initLines + 520, uploaded.size() + dropped); + + push(heldGather("gather_hold", "e", 9999)); + assertGathering("gather_hold", "e", ModuleConfiguration.logGatheringMaxBufferedLines); + int heldBefore = logger.heldLogLineCount(); + for (int i = 0; i < 600; i++) { + logger.e("[Test] second line " + i); + } + push(new ServerConfigBuilder().tracking(true).logGatheringOn("gather_hold", "e", 9999)); + logger.flushGatheredLogs(); + awaitSentBatches(); + + List later = receivedBatches(requests).subList(20, receivedBatches(requests).size()); + Assert.assertEquals(1, later.size()); + Assert.assertEquals(heldBefore + 100, later.get(0).getInt("d")); + Assert.assertEquals(numbered("[Test] second line ", 100, 600), messagesOf(later.get(0))); + } finally { + Countly.instance().halt(); + httpServer.stop(0); + } + } + + /** + * Every way a gather ends leaves nothing held, and a new gather never carries the lines of an + * earlier one. + *

+ * Verifies, with tracking off so nothing leaves until the end, that the same directive repeated + * and a change of levels or batch size within one gather keep the held lines; that + * {@code "e":false}, a directive without a usable gather id and a live response without any + * directive each stop the capture and release the delivery thread, their held lines offered to + * the request queue once, which refuses them while tracking is off; that a new gather id drops + * the lines of the previous gather; and that only the lines of the gather in effect, with its id, + * are ever uploaded. + */ + @Test + public void endedOrReplacedGather_leavesNothingHeld_andANewGatherNeverCarriesTheOldOnes() throws Exception { + server.respondWith(heldGather("gather_a", "e", 10)); + init(TestUtils.getConfigSdkBehaviorSettings()); + Log logger = logger(); + int initLines = logger.heldLogLineCount(); + logger.e("[Test] a1"); + logger.e("[Test] a2"); + Assert.assertEquals(initLines + 2, logger.heldLogLineCount()); + + push(heldGather("gather_a", "e", 10)); + Assert.assertEquals(initLines + 2, logger.heldLogLineCount()); + push(heldGather("gather_a", "ew", 20)); + assertGathering("gather_a", "ew", 20); + Assert.assertTrue(logger.heldLogLineCount() >= initLines + 2); + push(heldGather("gather_a", "e", 10)); + assertGathering("gather_a", "e", 10); + Assert.assertEquals(initLines + 2, logger.heldLogLineCount()); + + pushStop(new ServerConfigBuilder().networking(false).tracking(false).logGatheringOff()); + assertNotGathering(); + logger.e("[Test] logged while off"); + Assert.assertEquals(0, logger.heldLogLineCount()); + + push(heldGather("gather_b", "e", 10)); + assertGathering("gather_b", "e", 10); + Assert.assertEquals(0, logger.heldLogLineCount()); + logger.e("[Test] b1"); + push(heldGather("gather_c", "e", 10)); + assertGathering("gather_c", "e", 10); + Assert.assertEquals(0, logger.heldLogLineCount()); + logger.e("[Test] c1"); + + pushStop(new ServerConfigBuilder().networking(false).tracking(false).logGatheringDirective(true, " ", "e", 10)); + assertNotGathering(); + + push(heldGather("gather_d", "e", 10)); + logger.e("[Test] d1"); + pushStop(new ServerConfigBuilder().networking(false).tracking(false)); + assertNotGathering(); + + push(heldGather("gather_e", "e", 10)); + logger.e("[Test] e1"); + logger.e("[Test] e2"); + push(new ServerConfigBuilder().networking(false).tracking(true).logGatheringOn("gather_e", "e", 10)); + logger.flushGatheredLogs(); + awaitLogDelivery(); + + List batches = batches(); + Assert.assertEquals(1, batches.size()); + Assert.assertEquals("gather_e", batches.get(0).getString("i")); + Assert.assertEquals(0, batches.get(0).getInt("d")); + Assert.assertEquals(Arrays.asList("[Test] e1", "[Test] e2"), linesStartingWith("[Test] ", batches)); + } + + /** + * Only a live response decides. + *

+ * Verifies that an armed directive in the stored settings leaves the logger undecided, still + * capturing the init lines speculatively and queueing nothing even when flushed; that a run + * whose fetch fails decides off, lets go of what it held and captures nothing more; that a + * running gather survives a failed refresh with its lines; and that a stopped logger ignores + * every later line. + */ + @Test + public void onlyALiveResponseDecides_storedDirectiveStaysUndecided_andAFailedFetchDecidesOffOnlyWhileUndecided() throws Exception { + TestUtils.writeToFile(SDKStorage.JSON_FILE_NAME, new JSONObject() + .put("sc", new ServerConfigBuilder().networking(false).logGatheringOn("stored_gather_id", "ewidv", 40).build()).toString()); + server.pending(); + init(TestUtils.getConfigSdkBehaviorSettings()); + + Log undecided = logger(); + Assert.assertEquals(LogGatheringState.UNDECIDED, undecided.logGatheringState); + Assert.assertNull(undecided.logGatherId); + Assert.assertTrue(undecided.isCapturingLogs()); + Assert.assertFalse(undecided.isGatheringLogs()); + Assert.assertTrue(undecided.heldLogLineCount() > 0); + undecided.flushGatheredLogs(); + Assert.assertNull(undecided.logDeliveryExecutor); + Assert.assertTrue(logRequests().isEmpty()); + Countly.instance().stop(); + + Assert.assertEquals(LogGatheringState.NOT_GATHERING, undecided.logGatheringState); + undecided.e("[Test] logged after stop"); + Assert.assertEquals(0, undecided.heldLogLineCount()); + + server.failing(); + init(TestUtils.getConfigSdkBehaviorSettings()); + Log failed = logger(); + Assert.assertEquals(LogGatheringState.NOT_GATHERING, failed.logGatheringState); + Assert.assertEquals(0, failed.heldLogLineCount()); + failed.e("[Test] logged after the failed fetch"); + Assert.assertEquals(0, failed.heldLogLineCount()); + Countly.instance().halt(); + + server.respondWith(heldGather("gather_live", "e", 10)); + init(TestUtils.getConfigSdkBehaviorSettings()); + Log live = logger(); + live.e("[Test] kept across a failed refresh"); + int held = live.heldLogLineCount(); + Assert.assertTrue(held > 0); + server.failing(); + module().fetchConfigFromServer(SDKCore.instance.config); + assertGathering("gather_live", "e", 10); + Assert.assertEquals(held, live.heldLogLineCount()); + } + + /** + * While consent is required, a gathered line goes out only once the consent of the feature it + * belongs to is given, a line of no feature needs both the events and the user profiles + * consent, and the coordinates handed to a session are logged only with location consent. + *

+ * Verifies, with networking off, that the batch queued with only the sessions consent given + * holds no line of another feature and no coordinates; that the stop that follows events consent + * queues the events line but neither the location line nor the line of no feature; and that + * with location consent given the coordinates are gathered. + */ + @Test + public void requiredConsent_gathersEachLineOnlyWithTheConsentOfItsFeature() throws Exception { + Config config = TestUtils.getConfigSdkBehaviorSettings().setRequiresConsent(true) + .enableFeatures(Config.Feature.Events, Config.Feature.Sessions, Config.Feature.Location, Config.Feature.UserProfiles); + server.respondWith(new ServerConfigBuilder().networking(false).logGatheringOn("gather_consent", "ewidv", 500)); + init(config); + assertGathering("gather_consent", "ewidv", 500); + + Log logger = logger(); + logger.e("[Test] a line of no feature"); + logger.e("[ModuleEvents] an events line"); + logger.e("[ModuleLocation] a location line"); + Countly.onConsent(Config.Feature.Sessions); + Countly.session().begin(); + ((SessionImpl) Countly.session()).addLocation(12.345_678, 98.765_432); + logger.flushGatheredLogs(); + awaitLogDelivery(); + + List queued = receivedLines(batches()); + Assert.assertFalse(queued.isEmpty()); + Assert.assertFalse(queued.contains("[Test] a line of no feature")); + Assert.assertFalse(queued.contains("[ModuleEvents] an events line")); + Assert.assertFalse(queued.contains("[ModuleLocation] a location line")); + for (String line : queued) { + Assert.assertFalse(line, line.contains("12.345678")); + } + Assert.assertTrue(logger.awaitingConsentLineCount() >= 3); + + Countly.onConsent(Config.Feature.Events); + Countly.instance().stop(); + List stopped = receivedLines(storedBatchesOnDisk()); + Assert.assertTrue(stopped.contains("[ModuleEvents] an events line")); + Assert.assertFalse(stopped.contains("[ModuleLocation] a location line")); + Assert.assertFalse(stopped.contains("[Test] a line of no feature")); + for (String line : stopped) { + Assert.assertFalse(line, line.contains("12.345678")); + } + + server.respondWith(new ServerConfigBuilder().networking(false).logGatheringOn("gather_location", "ewidv", 500)); + init(config); + Countly.onConsent(Config.Feature.Sessions, Config.Feature.Location); + Countly.session().begin(); + ((SessionImpl) Countly.session()).addLocation(12.345_678, 98.765_432); + Countly.instance().stop(); + boolean coordinatesGathered = false; + for (JSONObject batch : storedBatchesOnDisk()) { + if ("gather_location".equals(batch.getString("i"))) { + for (String line : messagesOf(batch)) { + coordinatesGathered |= line.contains("12.345678"); + } + } + } + Assert.assertTrue(coordinatesGathered); + } + + /** + * While consent is required, a feature the app did not enable holds no line back, and a removed + * consent drops the held lines of its feature. + *

+ * Verifies, with networking off and the user profiles feature not enabled, that a line of no + * feature is gathered once the events and sessions consents are given; and that after the + * events consent is removed the stop queues the held sessions line but not the held events line. + */ + @Test + public void requiredConsent_aFeatureNotEnabledHoldsNoLineBack_andARemovedConsentDropsItsLines() throws Exception { + Config config = TestUtils.getConfigSdkBehaviorSettings().setRequiresConsent(true) + .enableFeatures(Config.Feature.Events, Config.Feature.Sessions); + server.respondWith(new ServerConfigBuilder().networking(false).logGatheringOn("gather_enabled", "e", 500)); + init(config); + Log logger = logger(); + logger.e("[Test] a line of no feature"); + Countly.onConsent(Config.Feature.Events, Config.Feature.Sessions); + logger.flushGatheredLogs(); + awaitLogDelivery(); + Assert.assertTrue(receivedLines(batches()).contains("[Test] a line of no feature")); + + logger.e("[ModuleEvents] a held events line"); + logger.e("[SessionImpl] a held sessions line"); + Countly.onConsentRemoval(Config.Feature.Events); + Countly.instance().stop(); + + List stopped = receivedLines(storedBatchesOnDisk()); + Assert.assertTrue(stopped.contains("[SessionImpl] a held sessions line")); + Assert.assertFalse(stopped.contains("[ModuleEvents] a held events line")); + } + + /** + * The global timer flushes a batch that is not full, a stop queues what is held before the + * storage stops, and a halt discards it. Either way the logger ends for good. + *

+ * Verifies that a line below the batch size reaches a local server on a tick of a fast global + * timer; that, with networking off, {@code stop()} queues the held line synchronously, so it is + * on disk once stop returns, with the gather id; that the stopped logger captures and queues + * nothing more and its delivery thread ends; and that {@code halt()} queues nothing at all. + */ + @Test + public void timerTick_flushesAPartialBatch_stopQueuesWhatIsHeld_andHaltDiscardsIt() throws Exception { + List queuedLogRequests = new CopyOnWriteArrayList<>(); + LogCallback listener = (message, level) -> { + if (message != null && message.startsWith("New request ") && message.contains(Log.transportMarker)) { + queuedLogRequests.add(message); + } + }; + List requests = new CopyOnWriteArrayList<>(); + HttpServer httpServer = startRecordingServer(requests); + try { + server.respondWith(new ServerConfigBuilder().logGatheringOn("gather_tick", "e", 100)); + init(localConfig(httpServer).setLogListener(listener)); + Log logger = logger(); + awaitSentBatches(); + + //the timer ticks through the modules, so it is sped up only after init built them + CountlyTimer.TIMER_DELAY_MS = 50; + push(new ServerConfigBuilder().sessionUpdateInterval(30).logGatheringOn("gather_tick", "e", 100)); + logger.e("[Test] tick line"); + Assert.assertTrue("no tick flushed the partial batch", TestUtils.waitFor(10_000, () -> !linesStartingWith("[Test] tick line", receivedBatches(requests)).isEmpty())); + + CountlyTimer.TIMER_DELAY_MS = 0; + push(new ServerConfigBuilder().networking(false).sessionUpdateInterval(60).logGatheringOn("gather_tick", "e", 100)); + //a tick of the replaced fast timer may still be running + Thread.sleep(250); + awaitLogDelivery(); + logger.e("[Test] tail line"); + Assert.assertEquals(1, logger.heldLogLineCount()); + Set deliveryThreads = liveDeliveryThreads(); + Assert.assertFalse(deliveryThreads.isEmpty()); + int queuedBeforeStop = queuedLogRequests.size(); + + Countly.instance().stop(); + Assert.assertEquals(queuedBeforeStop + 1, queuedLogRequests.size()); + JSONObject tail = null; + for (JSONObject batch : storedBatchesOnDisk()) { + if (messagesOf(batch).contains("[Test] tail line")) { + tail = batch; + } + } + Assert.assertNotNull("stop did not queue the held line", tail); + Assert.assertEquals("gather_tick", tail.getString("i")); + Assert.assertEquals(Arrays.asList("[Test] tail line"), messagesOf(tail)); + + Assert.assertEquals(LogGatheringState.NOT_GATHERING, logger.logGatheringState); + logger.e("[Test] logged after stop"); + logger.flushGatheredLogs(); + Assert.assertEquals(0, logger.heldLogLineCount()); + Assert.assertNull(logger.logDeliveryExecutor); + Assert.assertEquals(queuedBeforeStop + 1, queuedLogRequests.size()); + Assert.assertTrue("the delivery thread outlived the SDK", TestUtils.waitFor(5_000, () -> { + for (Thread thread : deliveryThreads) { + if (thread.isAlive()) { + return false; + } + } + return true; + })); + + server.respondWith(new ServerConfigBuilder().networking(false).logGatheringOn("gather_halt", "e", 100)); + init(TestUtils.getConfigSdkBehaviorSettings().setLogListener(listener)); + Log halted = logger(); + assertGathering("gather_halt", "e", 100); + awaitLogDelivery(); + int heldAtStart = halted.heldLogLineCount(); + halted.e("[Test] halted line"); + Assert.assertEquals(heldAtStart + 1, halted.heldLogLineCount()); + int queuedBeforeHalt = queuedLogRequests.size(); + + Countly.instance().halt(); + Assert.assertEquals(queuedBeforeHalt, queuedLogRequests.size()); + Assert.assertEquals(LogGatheringState.NOT_GATHERING, halted.logGatheringState); + Assert.assertEquals(0, halted.heldLogLineCount()); + } finally { + Countly.instance().halt(); + httpServer.stop(0); + } + } + + /** + * The threads that upload a batch never gather what they log meanwhile, or every upload would + * report on itself, while the lines about ordinary requests are gathered. + *

+ * Verifies against a local server that, once uploads and an ordinary event request were sent, + * no uploaded line quotes a batch, comes from the delivery thread or reports the preparing, + * sending or completion of an upload, while the completion of the event request is reported. + */ + @Test + public void uploadThreads_neverGatherTheirOwnLines_whileLinesAboutOrdinaryRequestsAreGathered() throws Exception { + List requests = new CopyOnWriteArrayList<>(); + HttpServer httpServer = startRecordingServer(requests); + + List heard = new CopyOnWriteArrayList<>(); + try { + Config config = localConfig(httpServer) + .enableFeatures(Config.Feature.Events) + .setEventQueueSizeToSend(1) + .setLogListener((message, level) -> heard.add(message)); + server.respondWith(new ServerConfigBuilder().logGatheringOn("gather_net", "ewidv", 50)); + init(config); + + Assert.assertTrue("no upload arrived", TestUtils.waitFor(30_000, () -> !receivedBatches(requests).isEmpty())); + Assert.assertTrue("no upload completed", TestUtils.waitFor(30_000, () -> !completedRequests(heard, requestIds(heard, Log.transportMarker)).isEmpty())); + + Countly.instance().events().recordEvent("ordinary_event"); + Assert.assertTrue("the event request never arrived", TestUtils.waitFor(30_000, () -> requests.stream().anyMatch(request -> request.params.containsKey("events")))); + Assert.assertTrue("the event request never completed", TestUtils.waitFor(30_000, () -> !completedRequests(heard, requestIds(heard, "events=")).isEmpty())); + Set eventIds = completedRequests(heard, requestIds(heard, "events=")); + + logger().flushGatheredLogs(); + Assert.assertTrue("the completion of the event request was not uploaded", TestUtils.waitFor(30_000, () -> { + for (String line : receivedLines(receivedBatches(requests))) { + for (String eventId : eventIds) { + if (line.startsWith("[Networking] Request " + eventId + " sent?: true")) { + return true; + } + } + } + return false; + })); + + Set uploadIds = completedRequests(heard, requestIds(heard, Log.transportMarker)); + Assert.assertFalse(uploadIds.isEmpty()); + for (JSONObject batch : receivedBatches(requests)) { + Assert.assertEquals("gather_net", batch.getString("i")); + } + for (String line : receivedLines(receivedBatches(requests))) { + Assert.assertFalse(line, line.contains(Log.keySdkLogs)); + Assert.assertFalse(line, line.startsWith("[Log] deliverHeldLogBatches")); + for (String uploadId : uploadIds) { + Assert.assertFalse(line, line.contains("Request " + uploadId + " sent?")); + Assert.assertFalse(line, line.contains("for request[" + uploadId + "]")); + } + } + } finally { + Countly.instance().halt(); + httpServer.stop(0); + } + } + + /** + * An armed SDK that is otherwise idle uploads nothing about its own uploads. + *

+ * Verifies against a local server that once the lines logged before went out a tick uploads + * nothing, that a line logged then goes out in one batch on the next tick, and that the ticks + * after that upload nothing, as neither queueing nor sending a batch, nor the drain that + * follows it, is gathered. + */ + @Test + public void idleArmedSdk_uploadsNothingAboutItsOwnUploads_onTheTicksAfterAnUpload() throws Exception { + List requests = new CopyOnWriteArrayList<>(); + HttpServer httpServer = startRecordingServer(requests); + try { + server.respondWith(new ServerConfigBuilder().logGatheringOn("gather_idle", "ewidv", 100)); + init(localConfig(httpServer)); + + awaitIdleTicks(requests); + logger().e("[Test] idle probe"); + Assert.assertEquals(1, tickAndCountBatches(requests)); + List received = receivedBatches(requests); + Assert.assertTrue(messagesOf(received.get(received.size() - 1)).contains("[Test] idle probe")); + for (int tick = 0; tick < 3; tick++) { + Assert.assertEquals("tick " + tick, 0, tickAndCountBatches(requests)); + } + Assert.assertEquals(0, logger().heldLogLineCount()); + } finally { + Countly.instance().halt(); + httpServer.stop(0); + } + } + + /** + * While nothing is sent, one log batch waits in the request queue and the rest stays in the + * capped buffer; once networking is back on, the batches go out one after another until nothing + * is held. + *

+ * Verifies, with networking off, that full batches and flushes queue nothing while a batch + * waits, the lines staying held; then, with networking on against a local server, that every + * held line arrives once, in order, with never more than one batch stored at a time. + */ + @Test + public void networkingOff_queuesOneBatchAtATime_andTheRestFollowsOnceItIsSent() throws Exception { + List requests = new CopyOnWriteArrayList<>(); + HttpServer httpServer = startRecordingServer(requests); + try { + server.respondWith(new ServerConfigBuilder().networking(false).logGatheringOn("gather_offline", "e", 10)); + init(localConfig(httpServer)); + awaitLogDelivery(); + Log logger = logger(); + for (int i = 0; i < 30; i++) { + logger.e("[Test] offline line " + i); + } + for (int flush = 0; flush < 3; flush++) { + logger.flushGatheredLogs(); + awaitLogDelivery(); + } + Assert.assertEquals(1, storedBatchCount()); + Assert.assertTrue(logger.heldLogLineCount() >= 20); + + push(new ServerConfigBuilder().logGatheringOn("gather_offline", "e", 10)); + int mostStored = 0; + long end = System.currentTimeMillis() + 30_000; + while (System.currentTimeMillis() < end) { + mostStored = Math.max(mostStored, storedBatchCount()); + if (logger.heldLogLineCount() == 0 && storedBatchCount() == 0) { + break; + } + Thread.sleep(5); + } + awaitSentBatches(); + Assert.assertEquals(1, mostStored); + Assert.assertEquals(numbered("[Test] offline line ", 0, 30), linesStartingWith("[Test] offline line ", receivedBatches(requests))); + } finally { + Countly.instance().halt(); + httpServer.stop(0); + } + } + + /** + * A device ID change without merge queues the gathered lines under the previous device ID and + * stops the gather, an answer requested for a previous device ID does not arm it again, and a + * change with merge keeps a gather running. + *

+ * Verifies, with networking off, that the line logged before the change is stored before the + * change returns, under the previous device ID and with the gather id; that nothing logged + * afterwards is held; that a settings answer requested before a later change and applied after + * it leaves the gather stopped while its other settings apply; that an answer for the current + * device ID arms a gather again; that a change with merge leaves that gather running; and that + * an answer requested before a change with merge arms its gather. + */ + @Test + public void deviceIdChangeWithoutMerge_queuesTheLinesUnderThePreviousId_andStopsTheGather() throws Exception { + server.respondWith(new ServerConfigBuilder().networking(false).logGatheringOn("gather_user_a", "e", 100)); + init(TestUtils.getConfigSdkBehaviorSettings()); + awaitLogDelivery(); + logger().e("[Test] line of user A"); + Countly.instance().deviceId().changeWithoutMerge("user_B"); + + Map userA = null; + for (Map request : logRequests()) { + if (request.get(Log.keySdkLogs).contains("[Test] line of user A")) { + userA = request; + } + } + Assert.assertNotNull("the line of user A was not queued before the change", userA); + Assert.assertEquals(TestUtils.DEVICE_ID, userA.get("device_id")); + Assert.assertEquals("gather_user_a", new JSONObject(userA.get(Log.keySdkLogs)).getString("i")); + assertNotGathering(); + logger().e("[Test] line of user B"); + Assert.assertEquals(0, logger().heldLogLineCount()); + + AtomicReference heldAnswer = new AtomicReference<>(); + InternalConfig config = SDKCore.instance.config; + config.immediateRequestGenerator = () -> (requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log) -> heldAnswer.set(callback); + module().fetchConfigFromServer(config); + Assert.assertNotNull(heldAnswer.get()); + Countly.instance().deviceId().changeWithoutMerge("user_C"); + heldAnswer.get().callback(new JSONObject(new ServerConfigBuilder().networking(false).sessionUpdateInterval(45).logGatheringOn("gather_stale", "e", 100).build())); + assertNotGathering(); + Assert.assertEquals(45, module().getSessionUpdateInterval()); + + config.immediateRequestGenerator = server; + push(new ServerConfigBuilder().networking(false).logGatheringOn("gather_user_c", "e", 100)); + assertGathering("gather_user_c", "e", 100); + Countly.instance().deviceId().changeWithMerge("user_D"); + assertGathering("gather_user_c", "e", 100); + + config.immediateRequestGenerator = () -> (requestData, customEndpoint, transport, requestShouldBeDelayed, networkingIsEnabled, callback, log) -> heldAnswer.set(callback); + module().fetchConfigFromServer(config); + Countly.instance().deviceId().changeWithMerge("user_E"); + heldAnswer.get().callback(new JSONObject(new ServerConfigBuilder().networking(false).logGatheringOn("gather_after_merge", "e", 100).build())); + assertGathering("gather_after_merge", "e", 100); + } + + /** + * A gather the server stops queues the lines it held once, with its id, and a gather replaced + * by a new id drops them. + *

+ * Verifies, with networking off and tracking on, that after {@code "e":false} exactly the lines + * held until then are stored, in one batch with the stopped gather's id, past the batch already + * waiting, and nothing logged afterwards; and that a gather replaced by a new id holds none of + * the lines of the one before. + */ + @Test + public void stoppedGather_queuesItsHeldLinesOnce_whileAReplacedGatherDropsThem() throws Exception { + server.respondWith(new ServerConfigBuilder().networking(false).logGatheringOn("gather_stopped", "e", 100)); + init(TestUtils.getConfigSdkBehaviorSettings()); + awaitLogDelivery(); + logger().e("[Test] stopped 1"); + logger().e("[Test] stopped 2"); + + push(new ServerConfigBuilder().networking(false).logGatheringOff()); + assertNotGathering(); + Assert.assertTrue("the held lines were not queued", TestUtils.waitFor(10_000, () -> linesStartingWith("[Test] stopped ", batches()).size() == 2)); + JSONObject tail = null; + for (JSONObject batch : batches()) { + if (messagesOf(batch).contains("[Test] stopped 1")) { + tail = batch; + } + } + Assert.assertNotNull(tail); + Assert.assertEquals("gather_stopped", tail.getString("i")); + Assert.assertEquals(Arrays.asList("[Test] stopped 1", "[Test] stopped 2"), linesStartingWith("[Test] stopped ", Arrays.asList(tail))); + logger().e("[Test] after the stop"); + Thread.sleep(200); + Assert.assertTrue(linesStartingWith("[Test] after the stop", batches()).isEmpty()); + + push(new ServerConfigBuilder().networking(false).logGatheringOn("gather_first", "e", 100)); + logger().e("[Test] first gather line"); + Assert.assertTrue(logger().heldLogLineCount() >= 1); + push(new ServerConfigBuilder().networking(false).logGatheringOn("gather_second", "e", 100)); + assertGathering("gather_second", "e", 100); + Assert.assertEquals(0, logger().heldLogLineCount()); + } + + /** + * Queueing a batch that is given up while the storage is busy cancels its store. + *

+ * Verifies, with the storage thread held busy, that a queueing interrupted while it waits for + * the store reports the batch as not queued, and that the store never runs once the storage is + * free again, so the batch is not stored behind the lines kept for another batch. + */ + @Test + public void queueingABatch_givenUpWhileTheStorageIsBusy_cancelsItsStore() throws Exception { + server.respondWith(new ServerConfigBuilder().networking(false).logGatheringOff()); + init(TestUtils.getConfigSdkBehaviorSettings()); + InternalConfig config = SDKCore.instance.config; + Storage.await(mock(Log.class)); + int storedBefore = logRequests().size(); + + CountDownLatch storageBusy = new CountDownLatch(1); + CountDownLatch storageFree = new CountDownLatch(1); + Storage.removeAsync(config, new Request(1L), removed -> { + storageBusy.countDown(); + storageFree.await(30, TimeUnit.SECONDS); + }); + AtomicReference queuedBatch = new AtomicReference<>(-1L); + Thread queueing = new Thread(() -> queuedBatch.set(ModuleRequests.pushSdkLogs(config, "{\"i\":\"gather_cancel\",\"d\":0,\"l\":[]}"))); + try { + Assert.assertTrue(storageBusy.await(30, TimeUnit.SECONDS)); + queueing.start(); + Assert.assertTrue("the queueing never waited for its store", TestUtils.waitFor(30_000, () -> isWaitingForAFuture(queueing))); + queueing.interrupt(); + queueing.join(30_000); + } finally { + storageFree.countDown(); + } + + Assert.assertNull(queuedBatch.get()); + Storage.await(mock(Log.class)); + Assert.assertEquals(storedBefore, logRequests().size()); + } + + // endregion + // region helpers + + /** + * A request a local server received. + */ + private static final class ReceivedRequest { + final String method; + final Map params; + + /** + * Holds one request. + * + * @param method the HTTP method it came with + * @param params its parameters, from the query and the body + */ + ReceivedRequest(String method, Map params) { + this.method = method; + this.params = params; + } + } + + /** + * Starts a local server that accepts every request and records it. + * + * @param requests where the requests go, in arrival order + * @return the started server + * @throws IOException when the server cannot start + */ + private static HttpServer startRecordingServer(List requests) throws IOException { + HttpServer httpServer = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + httpServer.createContext("/", exchange -> { + String query = exchange.getRequestURI().getRawQuery(); + requests.add(new ReceivedRequest(exchange.getRequestMethod(), decode((query == null ? "" : query) + "&" + readAll(exchange.getRequestBody())))); + byte[] body = "{\"result\":\"Success\"}".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(body); + } + }); + httpServer.start(); + return httpServer; + } + + /** + * The log batches among the requests a local server received. + * + * @param requests the requests, in arrival order + * @return the batches, in arrival order + */ + private static List receivedBatches(List requests) { + List received = new ArrayList<>(); + for (ReceivedRequest request : requests) { + if (request.params.containsKey(Log.keySdkLogs)) { + received.add(new JSONObject(request.params.get(Log.keySdkLogs))); + } + } + return received; + } + + /** + * The test configuration, sending to a local server instead of the test server. + * + * @param httpServer the local server + * @return the configuration + */ + private static Config localConfig(HttpServer httpServer) { + return new Config("http://localhost:" + httpServer.getAddress().getPort(), TestUtils.SERVER_APP_KEY, TestUtils.getTestSDirectory()) + .setApplicationVersion(TestUtils.APPLICATION_VERSION) + .setCustomDeviceId(TestUtils.DEVICE_ID) + .setNetworkRequestCooldown(0); + } + + /** + * Waits until every batch the logger queued was sent and no delivery is left to run, as each + * sent batch lets the next one go. + * + * @throws Exception when the batches are not all sent in time + */ + private static void awaitSentBatches() throws Exception { + for (int round = 0; round < 200; round++) { + awaitLogDelivery(); + Assert.assertTrue("a queued batch never reached the server", TestUtils.waitFor(30_000, () -> storedBatchCount() == 0)); + awaitLogDelivery(); + if (storedBatchCount() == 0) { + return; + } + } + Assert.fail("the logger kept queueing batches"); + } + + /** + * The log batches stored on disk, read straight from the request files, which works once the + * SDK stopped too. + * + * @return the batches + * @throws IOException when a request file cannot be read + */ + private static List storedBatchesOnDisk() throws IOException { + List stored = new ArrayList<>(); + for (String params : storedRequestParams()) { + Map decoded = decode(params); + if (decoded.containsKey(Log.keySdkLogs)) { + stored.add(new JSONObject(decoded.get(Log.keySdkLogs))); + } + } + return stored; + } + + /** + * Ticks until a tick queues no batch, which an idle SDK reaches once the lines it logged before + * went out, as nothing it logs about a batch is gathered. + * + * @param requests the requests a local server received + * @throws Exception when a batch does not go out in time + */ + private static void awaitIdleTicks(List requests) throws Exception { + for (int tick = 0; tick < 10; tick++) { + if (tickAndCountBatches(requests) == 0) { + return; + } + } + Assert.fail("every tick queued another batch, so each batch reported on the one before"); + } + + /** + * Flushes what is held, as a timer tick does, waits until the batches it queued are sent, then + * lets the drain that follows them run. + * + * @param requests the requests a local server received + * @return how many batches the tick queued + * @throws Exception when a batch does not go out in time + */ + private static int tickAndCountBatches(List requests) throws Exception { + int before = settledBatchCount(requests); + logger().flushGatheredLogs(); + awaitLogDelivery(); + int after = settledBatchCount(requests); + //nothing tells when the drain a stored batch asked for has finished, so it gets a moment + Thread.sleep(200); + return after - before; + } + + /** + * How many batches a local server received, counted once every stored one was sent. + * + * @param requests the requests a local server received + * @return the number of batches + * @throws InterruptedException when interrupted while waiting + */ + private static int settledBatchCount(List requests) throws InterruptedException { + Assert.assertTrue("a queued batch never reached the server", TestUtils.waitFor(30_000, () -> storedBatchCount() == 0)); + return receivedBatches(requests).size(); + } + + /** + * How many stored requests carry a log batch, read straight from disk, skipping a request the + * queue removes while it is read. + * + * @return the number of requests + */ + private static int storedBatchCount() { + File[] files = TestUtils.getTestSDirectory().listFiles((directory, name) -> name.startsWith(REQUEST_FILE_PREFIX)); + int count = 0; + for (File file : files == null ? new File[0] : files) { + try { + if (new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8).contains(Log.transportMarker)) { + count++; + } + } catch (IOException ignored) { + //sent and removed while it was read + } + } + return count; + } + + /** + * Whether a thread is waiting for the result of a future. + * + * @param thread the thread + * @return {@code true} while it waits in {@link java.util.concurrent.Future#get} + */ + private static boolean isWaitingForAFuture(Thread thread) { + for (StackTraceElement frame : thread.getStackTrace()) { + if ("java.util.concurrent.FutureTask".equals(frame.getClassName()) && "get".equals(frame.getMethodName())) { + return true; + } + } + return false; + } + + /** + * Initializes the SDK with {@link #server} answering its settings requests. + * + * @param config the configuration + */ + private void init(Config config) { + InternalConfig internalConfig = new InternalConfig(config); + internalConfig.immediateRequestGenerator = server; + Countly.instance().init(internalConfig); + } + + /** + * Answers the next settings request with the given payload and makes the SDK fetch, as the + * refresh timer does. + * + * @param builder the payload + */ + private void push(ServerConfigBuilder builder) { + server.respondWith(builder); + module().fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * Answers the next settings request with a payload that stops the gather, makes the SDK fetch, + * and waits until the delivery thread offered the last batch of the stopped gather to the + * request queue and ended, so the settings that follow cannot change how the queue answers. + * + * @param builder the payload + * @throws InterruptedException when interrupted while waiting + */ + private void pushStop(ServerConfigBuilder builder) throws InterruptedException { + ExecutorService executor = logger().logDeliveryExecutor; + push(builder); + if (executor != null) { + Assert.assertTrue("the delivery thread did not end", executor.awaitTermination(30, TimeUnit.SECONDS)); + } + } + + /** + * An armed directive next to settings that switch tracking off, so the gathered lines are held, + * and networking off, so nothing queued is sent. + * + * @param gatherId {@code i} + * @param levels {@code l} + * @param batchSize {@code b} + * @return the payload + */ + private static ServerConfigBuilder heldGather(String gatherId, String levels, int batchSize) { + return new ServerConfigBuilder().networking(false).tracking(false).logGatheringOn(gatherId, levels, batchSize); + } + + /** + * The configuration module of the running SDK. + * + * @return the module + */ + private static ModuleConfiguration module() { + return SDKCore.instance.module(ModuleConfiguration.class); + } + + /** + * The logger of the running SDK. + * + * @return the logger + */ + private static Log logger() { + return SDKCore.instance.L; + } + + /** + * Waits until every delivery the logger was asked for so far ran, and until the storage wrote + * every request it was handed. + * + * @throws Exception when the delivery thread does not finish in time + */ + private static void awaitLogDelivery() throws Exception { + ExecutorService executor = logger().logDeliveryExecutor; + if (executor != null) { + executor.submit(() -> { + }).get(30, TimeUnit.SECONDS); + } + Storage.await(mock(Log.class)); + } + + /** + * Asserts the directive twice over: as the configuration module decided it and as the logger + * mirrors it, which is what decides the capture. + * + * @param gatherId the gather id + * @param levels the level characters + * @param batchSize the batch size + */ + private static void assertGathering(String gatherId, String levels, int batchSize) { + ConfigurationProvider provider = SDKCore.instance.config.getConfigurationProvider(); + Assert.assertEquals(LogGatheringState.GATHERING, provider.getLogGatheringState()); + Assert.assertEquals(gatherId, provider.getLogGatheringId()); + Assert.assertEquals(levels, provider.getLogGatheringLevels()); + Assert.assertEquals(batchSize, provider.getLogGatheringBatchSize()); + + Log logger = logger(); + Assert.assertEquals(LogGatheringState.GATHERING, logger.logGatheringState); + Assert.assertTrue(logger.isGatheringLogs()); + Assert.assertTrue(logger.isCapturingLogs()); + Assert.assertEquals(gatherId, logger.logGatherId); + Assert.assertEquals(levels, logger.logGatherLevels); + Assert.assertEquals(batchSize, logger.logGatherBatchSize); + } + + /** + * Asserts that a decision against gathering was mirrored: nothing is captured or held, no + * gather id is left and no delivery thread is kept. + */ + private static void assertNotGathering() { + ConfigurationProvider provider = SDKCore.instance.config.getConfigurationProvider(); + Assert.assertEquals(LogGatheringState.NOT_GATHERING, provider.getLogGatheringState()); + Assert.assertNull(provider.getLogGatheringId()); + + Log logger = logger(); + Assert.assertEquals(LogGatheringState.NOT_GATHERING, logger.logGatheringState); + Assert.assertFalse(logger.isGatheringLogs()); + Assert.assertFalse(logger.isCapturingLogs()); + Assert.assertNull(logger.logGatherId); + Assert.assertEquals(ModuleConfiguration.logGatheringAllLevels, logger.logGatherLevels); + Assert.assertEquals(ModuleConfiguration.logGatheringDefaultBatchSize, logger.logGatherBatchSize); + Assert.assertEquals(0, logger.heldLogLineCount()); + Assert.assertNull(logger.logDeliveryExecutor); + } + + /** + * The queued requests that carry a log batch, oldest first. + * + * @return the parameters of each request + */ + private static List> logRequests() { + List> requests = new ArrayList<>(); + for (Map request : TestUtils.getCurrentRQ()) { + if (request.containsKey(Log.keySdkLogs)) { + requests.add(request); + } + } + return requests; + } + + /** + * The queued log batches, oldest first. + * + * @return the batches + */ + private static List batches() { + List batches = new ArrayList<>(); + for (Map request : logRequests()) { + batches.add(new JSONObject(request.get(Log.keySdkLogs))); + } + return batches; + } + + /** + * The messages of a batch, in order. + * + * @param batch the batch + * @return the messages + */ + private static List messagesOf(JSONObject batch) { + List messages = new ArrayList<>(); + JSONArray lines = batch.getJSONArray("l"); + for (int i = 0; i < lines.length(); i++) { + messages.add(lines.getJSONObject(i).getString("m")); + } + return messages; + } + + /** + * The messages of every batch that start with the given prefix, in upload order. + * + * @param prefix the prefix + * @param batches the batches + * @return the messages + */ + private static List linesStartingWith(String prefix, List batches) { + List messages = new ArrayList<>(); + for (JSONObject batch : batches) { + for (String message : messagesOf(batch)) { + if (message.startsWith(prefix)) { + messages.add(message); + } + } + } + return messages; + } + + /** + * The messages of every batch a server received. + * + * @param received the batches, in arrival order + * @return the messages + */ + private static List receivedLines(List received) { + List messages = new ArrayList<>(); + for (JSONObject batch : new ArrayList<>(received)) { + messages.addAll(messagesOf(batch)); + } + return messages; + } + + /** + * The storage IDs of the queued requests whose parameters contain the given text, read off + * the lines the SDK logs as it queues a request. + * + * @param heard every line the SDK logged + * @param paramsFragment the text + * @return the IDs + */ + private static Set requestIds(List heard, String paramsFragment) { + Pattern queued = Pattern.compile("^New request (\\d+): "); + Set ids = new HashSet<>(); + for (String line : heard) { + Matcher matcher = queued.matcher(line == null ? "" : line); + if (matcher.find() && line.contains(paramsFragment)) { + ids.add(matcher.group(1)); + } + } + return ids; + } + + /** + * The given requests the SDK reported as sent and accepted. + * + * @param heard every line the SDK logged + * @param ids storage IDs of requests + * @return the IDs of those that completed + */ + private static Set completedRequests(List heard, Set ids) { + Set completed = new HashSet<>(); + for (String line : heard) { + for (String id : ids) { + if (line != null && line.startsWith("[Networking] Request " + id + " sent?: true")) { + completed.add(id); + } + } + } + return completed; + } + + /** + * The live delivery threads of every logger. + * + * @return the threads + */ + private static Set liveDeliveryThreads() { + Set threads = new HashSet<>(); + for (Thread thread : Thread.getAllStackTraces().keySet()) { + if (Log.DELIVERY_THREAD_NAME.equals(thread.getName()) && thread.isAlive()) { + threads.add(thread); + } + } + return threads; + } + + /** + * The parameters of every stored request, oldest first, read straight from disk, so they can be + * read after the storage stopped. + * + * @return the url encoded parameters of each request + * @throws IOException when a request cannot be read + */ + private static List storedRequestParams() throws IOException { + File[] files = TestUtils.getTestSDirectory().listFiles((directory, name) -> name.startsWith(REQUEST_FILE_PREFIX)); + List sorted = new ArrayList<>(Arrays.asList(files == null ? new File[0] : files)); + sorted.sort(Comparator.comparing(file -> Long.parseLong(file.getName().substring(REQUEST_FILE_PREFIX.length())))); + + List params = new ArrayList<>(); + for (File file : sorted) { + String content = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + int end = content.indexOf('\n'); + params.add(end < 0 ? content : content.substring(0, end)); + } + return params; + } + + /** + * Decodes url encoded parameters. + * + * @param params the parameters + * @return the parameters by name + */ + private static Map decode(String params) { + Map decoded = new HashMap<>(); + for (String pair : params.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] keyValue = pair.split("=", 2); + decoded.put(Utils.urldecode(keyValue[0]), keyValue.length > 1 ? Utils.urldecode(keyValue[1]) : ""); + } + return decoded; + } + + /** + * Reads a stream to its end. + * + * @param stream the stream + * @return its content as UTF-8 text + * @throws IOException when it cannot be read + */ + private static String readAll(InputStream stream) throws IOException { + ByteArrayOutputStream content = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = stream.read(buffer)) != -1) { + content.write(buffer, 0, read); + } + return new String(content.toByteArray(), StandardCharsets.UTF_8); + } + + /** + * Numbered lines, a prefix followed by each number in a range. + * + * @param prefix the prefix + * @param from the first number + * @param to one past the last number + * @return the lines + */ + private static List numbered(String prefix, int from, int to) { + List lines = new ArrayList<>(); + for (int i = from; i < to; i++) { + lines.add(prefix + i); + } + return lines; + } + + /** + * A text of one character repeated. + * + * @param character the character + * @param count how many times + * @return the text + */ + private static String repeat(char character, int count) { + StringBuilder text = new StringBuilder(count); + for (int i = 0; i < count; i++) { + text.append(character); + } + return text.toString(); + } + + + // endregion +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ServerConfigBuilder.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ServerConfigBuilder.java new file mode 100644 index 00000000..745e8a2c --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ServerConfigBuilder.java @@ -0,0 +1,709 @@ +package ly.count.sdk.java.internal; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import org.json.JSONObject; +import org.junit.Assert; + +import static ly.count.sdk.java.internal.ModuleConfiguration.keyLGBatchSize; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyLGEnabled; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyLGId; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyLGLevels; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRAutomaticCrashReporting; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRAutomaticSessionTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRAutomaticViewTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRBOMAcceptedTimeout; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRBOMDuration; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRBOMRQPercentage; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRBOMRequestAge; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRBackoffMechanism; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRConfig; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRConnectionTest; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRConsentRequired; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRContentZoneInterval; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRCrashReporting; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRCustomEventTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRDropOldRequestTime; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREnterContentZone; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREventBlacklist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREventQueueSize; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREventSegmentationBlacklist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREventSegmentationWhitelist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyREventWhitelist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRJourneyTriggerEvents; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRJourneyTriggerViews; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitBreadcrumb; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitKeyLength; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitSegValues; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitTraceLength; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitTraceLine; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLimitValueSize; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLocationTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLogGathering; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRLogging; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRNetworking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRRefreshContentZone; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRReqQueueSize; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRSegmentationBlacklist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRSegmentationWhitelist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRServerConfigUpdateInterval; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRSessionTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRSessionUpdateInterval; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRTimestamp; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRTracking; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRUserPropertyBlacklist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRUserPropertyCacheLimit; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRUserPropertyWhitelist; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRVersion; +import static ly.count.sdk.java.internal.ModuleConfiguration.keyRViewTracking; + +/** + * Builds SDK behavior settings payloads, {@code {"v":..,"t":..,"c":{..}}} plus any sibling of + * {@code c}, and checks a {@link ConfigurationProvider} against the settings it holds. + */ +class ServerConfigBuilder { + final Map config = new LinkedHashMap<>(); + /** + * Keys that sit next to {@code c}, not inside it: the {@code lg} directive, the {@code ct} + * flag and anything a newer server may add. + */ + final Map topLevelKeys = new LinkedHashMap<>(); + private Object timestamp = System.currentTimeMillis(); + private Object version = 1; + + /** + * Sets the {@code t} value. + * + * @param timestamp the value + * @return this builder + */ + ServerConfigBuilder timestamp(Object timestamp) { + this.timestamp = timestamp; + return this; + } + + /** + * Sets the {@code v} value. + * + * @param version the value + * @return this builder + */ + ServerConfigBuilder version(Object version) { + this.version = version; + return this; + } + + /** + * Sets any setting to any value, including the ones the SDK has to reject. + * + * @param key the settings key + * @param value the value + * @return this builder + */ + ServerConfigBuilder set(String key, Object value) { + config.put(key, value); + return this; + } + + /** + * @param enabled {@code tracking} + * @return this builder + */ + ServerConfigBuilder tracking(boolean enabled) { + return set(keyRTracking, enabled); + } + + /** + * @param enabled {@code networking} + * @return this builder + */ + ServerConfigBuilder networking(boolean enabled) { + return set(keyRNetworking, enabled); + } + + /** + * @param enabled {@code st} + * @return this builder + */ + ServerConfigBuilder sessionTracking(boolean enabled) { + return set(keyRSessionTracking, enabled); + } + + /** + * @param enabled {@code vt} + * @return this builder + */ + ServerConfigBuilder viewTracking(boolean enabled) { + return set(keyRViewTracking, enabled); + } + + /** + * @param enabled {@code cet} + * @return this builder + */ + ServerConfigBuilder customEventTracking(boolean enabled) { + return set(keyRCustomEventTracking, enabled); + } + + /** + * @param enabled {@code ecz} + * @return this builder + */ + ServerConfigBuilder contentZone(boolean enabled) { + return set(keyREnterContentZone, enabled); + } + + /** + * @param enabled {@code crt} + * @return this builder + */ + ServerConfigBuilder crashReporting(boolean enabled) { + return set(keyRCrashReporting, enabled); + } + + /** + * @param enabled {@code ast} + * @return this builder + */ + ServerConfigBuilder automaticSessionTracking(boolean enabled) { + return set(keyRAutomaticSessionTracking, enabled); + } + + /** + * @param enabled {@code avt} + * @return this builder + */ + ServerConfigBuilder automaticViewTracking(boolean enabled) { + return set(keyRAutomaticViewTracking, enabled); + } + + /** + * @param enabled {@code acr} + * @return this builder + */ + ServerConfigBuilder automaticCrashReporting(boolean enabled) { + return set(keyRAutomaticCrashReporting, enabled); + } + + /** + * @param enabled {@code lt} + * @return this builder + */ + ServerConfigBuilder locationTracking(boolean enabled) { + return set(keyRLocationTracking, enabled); + } + + /** + * @param enabled {@code rcz} + * @return this builder + */ + ServerConfigBuilder refreshContentZone(boolean enabled) { + return set(keyRRefreshContentZone, enabled); + } + + /** + * @param enabled {@code bom} + * @return this builder + */ + ServerConfigBuilder backoffMechanism(boolean enabled) { + return set(keyRBackoffMechanism, enabled); + } + + /** + * @param enabled {@code log} + * @return this builder + */ + ServerConfigBuilder logging(boolean enabled) { + return set(keyRLogging, enabled); + } + + /** + * @param required {@code cr} + * @return this builder + */ + ServerConfigBuilder consentRequired(boolean required) { + return set(keyRConsentRequired, required); + } + + /** + * @param hours {@code scui} + * @return this builder + */ + ServerConfigBuilder serverConfigUpdateInterval(int hours) { + return set(keyRServerConfigUpdateInterval, hours); + } + + /** + * @param size {@code rqs} + * @return this builder + */ + ServerConfigBuilder requestQueueSize(int size) { + return set(keyRReqQueueSize, size); + } + + /** + * @param size {@code eqs} + * @return this builder + */ + ServerConfigBuilder eventQueueSize(int size) { + return set(keyREventQueueSize, size); + } + + /** + * @param seconds {@code sui} + * @return this builder + */ + ServerConfigBuilder sessionUpdateInterval(int seconds) { + return set(keyRSessionUpdateInterval, seconds); + } + + /** + * @param limit {@code lkl} + * @return this builder + */ + ServerConfigBuilder keyLengthLimit(int limit) { + return set(keyRLimitKeyLength, limit); + } + + /** + * @param limit {@code lvs} + * @return this builder + */ + ServerConfigBuilder valueSizeLimit(int limit) { + return set(keyRLimitValueSize, limit); + } + + /** + * @param limit {@code lsv} + * @return this builder + */ + ServerConfigBuilder segmentationValuesLimit(int limit) { + return set(keyRLimitSegValues, limit); + } + + /** + * @param limit {@code lbc} + * @return this builder + */ + ServerConfigBuilder breadcrumbLimit(int limit) { + return set(keyRLimitBreadcrumb, limit); + } + + /** + * @param limit {@code ltlpt} + * @return this builder + */ + ServerConfigBuilder traceLinesLimit(int limit) { + return set(keyRLimitTraceLine, limit); + } + + /** + * @param limit {@code ltl} + * @return this builder + */ + ServerConfigBuilder traceLengthLimit(int limit) { + return set(keyRLimitTraceLength, limit); + } + + /** + * @param limit {@code upcl} + * @return this builder + */ + ServerConfigBuilder userPropertyCacheLimit(int limit) { + return set(keyRUserPropertyCacheLimit, limit); + } + + /** + * @param seconds {@code bom_at} + * @return this builder + */ + ServerConfigBuilder backoffAcceptedTimeout(int seconds) { + return set(keyRBOMAcceptedTimeout, seconds); + } + + /** + * @param share {@code bom_rqp} + * @return this builder + */ + ServerConfigBuilder backoffRequestQueuePercentage(double share) { + return set(keyRBOMRQPercentage, share); + } + + /** + * @param hours {@code bom_ra} + * @return this builder + */ + ServerConfigBuilder backoffRequestAge(int hours) { + return set(keyRBOMRequestAge, hours); + } + + /** + * @param seconds {@code bom_d} + * @return this builder + */ + ServerConfigBuilder backoffDuration(int seconds) { + return set(keyRBOMDuration, seconds); + } + + /** + * @param hours {@code dort} + * @return this builder + */ + ServerConfigBuilder dropOldRequestTime(int hours) { + return set(keyRDropOldRequestTime, hours); + } + + /** + * @param seconds {@code czi} + * @return this builder + */ + ServerConfigBuilder contentZoneInterval(int seconds) { + return set(keyRContentZoneInterval, seconds); + } + + /** + * Sets {@code eb} or {@code ew}, dropping the other one. + * + * @param names the event keys + * @param isWhitelist {@code true} for {@code ew} + * @return this builder + */ + ServerConfigBuilder eventFilterList(Set names, boolean isWhitelist) { + return filter(keyREventBlacklist, keyREventWhitelist, names, isWhitelist); + } + + /** + * Sets {@code upb} or {@code upw}, dropping the other one. + * + * @param names the user property keys + * @param isWhitelist {@code true} for {@code upw} + * @return this builder + */ + ServerConfigBuilder userPropertyFilterList(Set names, boolean isWhitelist) { + return filter(keyRUserPropertyBlacklist, keyRUserPropertyWhitelist, names, isWhitelist); + } + + /** + * Sets {@code sb} or {@code sw}, dropping the other one. + * + * @param names the segmentation keys + * @param isWhitelist {@code true} for {@code sw} + * @return this builder + */ + ServerConfigBuilder segmentationFilterList(Set names, boolean isWhitelist) { + return filter(keyRSegmentationBlacklist, keyRSegmentationWhitelist, names, isWhitelist); + } + + /** + * Sets {@code esb} or {@code esw}, dropping the other one. + * + * @param namesPerEvent the segmentation keys of each event key + * @param isWhitelist {@code true} for {@code esw} + * @return this builder + */ + ServerConfigBuilder eventSegmentationFilterMap(Map> namesPerEvent, boolean isWhitelist) { + config.remove(isWhitelist ? keyREventSegmentationBlacklist : keyREventSegmentationWhitelist); + return set(isWhitelist ? keyREventSegmentationWhitelist : keyREventSegmentationBlacklist, namesPerEvent); + } + + /** + * @param eventKeys {@code jte} + * @return this builder + */ + ServerConfigBuilder journeyTriggerEvents(Set eventKeys) { + return set(keyRJourneyTriggerEvents, eventKeys); + } + + /** + * @param viewNames {@code jtv} + * @return this builder + */ + ServerConfigBuilder journeyTriggerViews(Set viewNames) { + return set(keyRJourneyTriggerViews, viewNames); + } + + /** + * Sets one of a blacklist and whitelist pair, dropping the other one. + * + * @param blacklistKey the blacklist key + * @param whitelistKey the whitelist key + * @param names the names + * @param isWhitelist whether to set the whitelist + * @return this builder + */ + private ServerConfigBuilder filter(String blacklistKey, String whitelistKey, Set names, boolean isWhitelist) { + config.remove(isWhitelist ? blacklistKey : whitelistKey); + return set(isWhitelist ? whitelistKey : blacklistKey, names); + } + + /** + * Sets any key next to {@code c}. + * + * @param key the top level key + * @param value the value + * @return this builder + */ + ServerConfigBuilder topLevelKey(String key, Object value) { + topLevelKeys.put(key, value); + return this; + } + + /** + * Sets the {@code ct} connection test flag. + * + * @param value the flag, of any type + * @return this builder + */ + ServerConfigBuilder connectionTest(Object value) { + return topLevelKey(keyRConnectionTest, value); + } + + /** + * A log gathering directive that turns gathering off, {@code {"e":false}}. + * + * @return this builder + */ + ServerConfigBuilder logGatheringOff() { + return logGatheringDirective(false, null, null, null); + } + + /** + * A log gathering directive with every field filled in. + * + * @param gatherId {@code i} + * @param levels {@code l} + * @param batchSize {@code b} + * @return this builder + */ + ServerConfigBuilder logGatheringOn(String gatherId, String levels, int batchSize) { + return logGatheringDirective(true, gatherId, levels, batchSize); + } + + /** + * A log gathering directive from raw values, so malformed ones can be built too. A {@code null} + * value is left out rather than written as JSON null. + * + * @param enabled {@code e} + * @param gatherId {@code i} + * @param levels {@code l} + * @param batchSize {@code b} + * @return this builder + */ + ServerConfigBuilder logGatheringDirective(Object enabled, Object gatherId, Object levels, Object batchSize) { + JSONObject directive = new JSONObject(); + putIfNotNull(directive, keyLGEnabled, enabled); + putIfNotNull(directive, keyLGId, gatherId); + putIfNotNull(directive, keyLGLevels, levels); + putIfNotNull(directive, keyLGBatchSize, batchSize); + return topLevelKey(keyRLogGathering, directive); + } + + /** + * Puts a value unless it is {@code null}. + * + * @param target the object to put into + * @param key the key + * @param value the value + */ + private static void putIfNotNull(JSONObject target, String key, Object value) { + if (value != null) { + target.put(key, value); + } + } + + /** + * Every setting at a valid value that differs from what the SDK uses by default. + * + * @return this builder + */ + ServerConfigBuilder allKeysAtNonDefaultValues() { + tracking(false).networking(false).sessionTracking(false).viewTracking(false).customEventTracking(false) + .contentZone(true).crashReporting(false).automaticSessionTracking(true).automaticViewTracking(true) + .automaticCrashReporting(false).locationTracking(false).refreshContentZone(false).backoffMechanism(false) + .logging(true).consentRequired(true); + + serverConfigUpdateInterval(8).requestQueueSize(2000).eventQueueSize(200).sessionUpdateInterval(120) + .keyLengthLimit(89).valueSizeLimit(43).segmentationValuesLimit(25).breadcrumbLimit(90) + .traceLinesLimit(89).traceLengthLimit(78).userPropertyCacheLimit(67) + .backoffAcceptedTimeout(12).backoffRequestQueuePercentage(0.25).backoffRequestAge(36).backoffDuration(90) + .dropOldRequestTime(5).contentZoneInterval(60); + + eventFilterList(names("blocked_event"), false); + userPropertyFilterList(names("allowed_property"), true); + segmentationFilterList(names("blocked_segment"), false); + Map> perEvent = new LinkedHashMap<>(); + perEvent.put("purchase", names("card_number")); + eventSegmentationFilterMap(perEvent, false); + journeyTriggerEvents(names("journey_event")); + return journeyTriggerViews(names("journey_view")); + } + + /** + * A set of names in the given order. + * + * @param names the names + * @return the set + */ + static Set names(String... names) { + return new LinkedHashSet<>(Arrays.asList(names)); + } + + /** + * The payload as the server sends it. + * + * @return the JSON text + */ + String build() { + return buildJson().toString(); + } + + /** + * The payload as a JSON object. + * + * @return the payload + */ + JSONObject buildJson() { + JSONObject json = new JSONObject(); + json.put(keyRVersion, version); + json.put(keyRTimestamp, timestamp); + json.put(keyRConfig, new JSONObject(config)); + for (Map.Entry entry : topLevelKeys.entrySet()) { + json.put(entry.getKey(), entry.getValue()); + } + return json; + } + + /** + * Asserts that the provider serves every setting this builder holds. + * + * @param provider the provider to check + */ + @SuppressWarnings("unchecked") + void validateAgainst(ConfigurationProvider provider) { + for (Map.Entry entry : config.entrySet()) { + String key = entry.getKey(); + Object expected = entry.getValue(); + switch (key) { + case keyREventBlacklist: + case keyREventWhitelist: + assertFilter(key, (Set) expected, keyREventWhitelist.equals(key), provider.getEventFilterList()); + break; + case keyRUserPropertyBlacklist: + case keyRUserPropertyWhitelist: + assertFilter(key, (Set) expected, keyRUserPropertyWhitelist.equals(key), provider.getUserPropertyFilterList()); + break; + case keyRSegmentationBlacklist: + case keyRSegmentationWhitelist: + assertFilter(key, (Set) expected, keyRSegmentationWhitelist.equals(key), provider.getSegmentationFilterList()); + break; + case keyREventSegmentationBlacklist: + case keyREventSegmentationWhitelist: + ConfigurationProvider.FilterList>> perEvent = provider.getEventSegmentationFilterList(); + Assert.assertEquals("kind of '" + key + "'", keyREventSegmentationWhitelist.equals(key), perEvent.isWhitelist()); + Assert.assertEquals("names of '" + key + "'", expected, perEvent.getFilterList()); + break; + default: + Assert.assertEquals("value of '" + key + "'", expected, valueOf(provider, key)); + break; + } + } + } + + /** + * Asserts one names filter. + * + * @param key the settings key + * @param expected the names + * @param isWhitelist the expected kind + * @param actual the filter the provider serves + */ + private static void assertFilter(String key, Set expected, boolean isWhitelist, ConfigurationProvider.FilterList> actual) { + Assert.assertEquals("kind of '" + key + "'", isWhitelist, actual.isWhitelist()); + Assert.assertEquals("names of '" + key + "'", expected, actual.getFilterList()); + } + + /** + * The value a provider serves for a setting that is not a listing filter. + * + * @param provider the provider + * @param key the settings key + * @return the served value, boxed + */ + static Object valueOf(ConfigurationProvider provider, String key) { + switch (key) { + case keyRTracking: + return provider.getTrackingEnabled(); + case keyRNetworking: + return provider.getNetworkingEnabled(); + case keyRSessionTracking: + return provider.getSessionTrackingEnabled(); + case keyRViewTracking: + return provider.getViewTrackingEnabled(); + case keyRCustomEventTracking: + return provider.getCustomEventTrackingEnabled(); + case keyREnterContentZone: + return provider.getContentZoneEnabled(); + case keyRCrashReporting: + return provider.getCrashReportingEnabled(); + case keyRAutomaticSessionTracking: + return provider.getAutomaticSessionTrackingEnabled(); + case keyRAutomaticViewTracking: + return provider.getAutomaticViewTrackingEnabled(); + case keyRAutomaticCrashReporting: + return provider.getAutomaticCrashReportingEnabled(); + case keyRLocationTracking: + return provider.getLocationTrackingEnabled(); + case keyRRefreshContentZone: + return provider.getRefreshContentZoneEnabled(); + case keyRBackoffMechanism: + return provider.getBOMEnabled(); + case keyRLogging: + return provider.getLoggingEnabled(); + case keyRConsentRequired: + return provider.getConsentRequired(); + case keyRServerConfigUpdateInterval: + return provider.getServerConfigUpdateInterval(); + case keyRReqQueueSize: + return provider.getRequestQueueMaxSize(); + case keyREventQueueSize: + return provider.getEventQueueSizeThreshold(); + case keyRSessionUpdateInterval: + return provider.getSessionUpdateInterval(); + case keyRLimitKeyLength: + return provider.getMaxKeyLength(); + case keyRLimitValueSize: + return provider.getMaxValueSize(); + case keyRLimitSegValues: + return provider.getMaxSegmentationValues(); + case keyRLimitBreadcrumb: + return provider.getMaxBreadcrumbCount(); + case keyRLimitTraceLine: + return provider.getMaxStackTraceLinesPerThread(); + case keyRLimitTraceLength: + return provider.getMaxStackTraceLineLength(); + case keyRUserPropertyCacheLimit: + return provider.getUserPropertyCacheLimit(); + case keyRBOMAcceptedTimeout: + return provider.getBOMAcceptedTimeoutSeconds(); + case keyRBOMRQPercentage: + return provider.getBOMRQPercentage(); + case keyRBOMRequestAge: + return provider.getBOMRequestAge(); + case keyRBOMDuration: + return provider.getBOMDuration(); + case keyRDropOldRequestTime: + return provider.getRequestDropAgeHours(); + case keyRContentZoneInterval: + return provider.getContentZoneTimerInterval(); + case keyRJourneyTriggerEvents: + return provider.getJourneyTriggerEvents(); + case keyRJourneyTriggerViews: + return provider.getJourneyTriggerViews(); + default: + throw new IllegalArgumentException("no getter for settings key [" + key + "]"); + } + } +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/TestUtils.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/TestUtils.java index 49197359..0d558d64 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/TestUtils.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/TestUtils.java @@ -13,6 +13,7 @@ import java.util.Scanner; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -57,11 +58,36 @@ static Config getBaseConfig() { } static Config getBaseConfig(String deviceID) { + Config config = getConfigSdkBehaviorSettings(deviceID); + // Disable server config requests by default in tests so the suite doesn't fire HTTP calls + // to a non-existent test host on every init. Tests covering ModuleConfiguration opt back in + // explicitly. + config.disableSdkBehaviorSettingsUpdates(); + return config; + } + + /** + * The base configuration with the SDK behavior settings requests left on, for the tests that + * exercise them. Pair it with an immediate request generator, or every init asks the test + * server for its settings. + * + * @return the configuration + */ + static Config getConfigSdkBehaviorSettings() { + return getConfigSdkBehaviorSettings(DEVICE_ID); + } + + /** + * The base configuration with the SDK behavior settings requests left on, for a given device ID. + * + * @param deviceID the custom device ID + * @return the configuration + */ + static Config getConfigSdkBehaviorSettings(String deviceID) { File sdkStorageRootDirectory = getTestSDirectory(); checkSdkStorageRootDirectoryExist(sdkStorageRootDirectory); Config config = new Config(SERVER_URL, SERVER_APP_KEY, sdkStorageRootDirectory); config.setApplicationVersion(APPLICATION_VERSION); - config.setCustomDeviceId(deviceID); return config; } @@ -425,6 +451,19 @@ static void validateEQSize(int expectedSize) { Assert.assertEquals(expectedSize, getCurrentEQ().size()); } + /** + * The keys of the events in the event queue, in queue order. + * + * @return the keys + */ + static List keysInEQ() { + List keys = new ArrayList<>(); + for (EventImpl event : getCurrentEQ()) { + keys.add(event.key); + } + return keys; + } + static String getOS() { return System.getProperty("os.name"); } @@ -698,4 +737,40 @@ static InternalConfig getConfigViews(Map segmentation) { config.views.setGlobalViewSegmentation(segmentation); return config; } + + /** + * Waits until the clock is past every timestamp handed out so far. A request file is named after + * a {@link TimeUtils#uniqueTimestampMs()} value, which is unique among the last ten values only, + * so requests stored back to back could reuse a name and overwrite each other. + */ + static void letTheClockCatchUp() { + long latest = TimeUtils.uniqueTimestampMs(); + while (System.currentTimeMillis() <= latest) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + /** + * Polls a condition until it holds or the time is up. + * + * @param timeoutMs how long to wait at most, in milliseconds + * @param condition the condition + * @return whether it held in time + * @throws InterruptedException when interrupted while waiting + */ + static boolean waitFor(long timeoutMs, BooleanSupplier condition) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return true; + } + Thread.sleep(10); + } + return condition.getAsBoolean(); + } } diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/UtilsInternalLimitsTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/UtilsInternalLimitsTests.java new file mode 100644 index 00000000..4f3ad28d --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/UtilsInternalLimitsTests.java @@ -0,0 +1,168 @@ +package ly.count.sdk.java.internal; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import ly.count.sdk.java.Config; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * The truncation rules of {@link UtilsInternalLimits} that the scenarios through the public API cannot + * reach on every platform: surrogate pairs, {@code \r\n} line breaks, {@code null} keys and the + * independence from the order of a map. + */ +@RunWith(JUnit4.class) +public class UtilsInternalLimitsTests { + + private static final String TAG = "[Caller] method"; + private final List warnings = new CopyOnWriteArrayList<>(); + private Log log; + + /** + * Starts every test with a logger that keeps every warning. + */ + @Before + public void beforeTest() { + warnings.clear(); + log = new Log(Config.LoggingLevel.OFF, (message, level) -> { + if (level == Config.LoggingLevel.WARN) { + warnings.add(message); + } + }); + } + + /** + * Keys and values are cut to the limit and never inside a surrogate pair. + *

+ * Verifies that {@code null}, empty and short strings come back as the same instance without a + * warning; that a longer string is cut to the limit with one warning naming the caller; that a + * cut falling inside a surrogate pair falls before it, or after it when the pair starts the + * string; and that only strings are cut by value, other types coming back as they are. + */ + @Test + public void strings_areCutToTheLimit_neverInsideASurrogatePair() { + String shortKey = "key"; + Assert.assertSame(shortKey, UtilsInternalLimits.truncateKey(shortKey, 3, log, TAG)); + Assert.assertNull(UtilsInternalLimits.truncateKey(null, 1, log, TAG)); + Assert.assertEquals("", UtilsInternalLimits.truncateValue("", 1, log, TAG)); + Assert.assertEquals(0, warnings.size()); + + Assert.assertEquals("abc", UtilsInternalLimits.truncateKey("abcdef", 3, log, TAG)); + Assert.assertEquals(1, warnings.size()); + Assert.assertTrue(warnings.get(0), warnings.get(0).startsWith(TAG + ", the key [abcdef] is longer than the key length limit of [3]")); + Assert.assertEquals("abcd", UtilsInternalLimits.truncateValue("abcdefgh", 4, log, TAG)); + Assert.assertTrue(warnings.get(1), warnings.get(1).startsWith(TAG + ", a value of [8] characters is longer than the value size limit of [4]")); + + String grinning = "😀"; + Assert.assertEquals("ab", UtilsInternalLimits.truncateValue("ab" + grinning + "cd", 3, log, TAG)); + Assert.assertEquals("ab" + grinning, UtilsInternalLimits.truncateValue("ab" + grinning + "cd", 4, log, TAG)); + Assert.assertEquals(grinning, UtilsInternalLimits.truncateKey(grinning + "rest", 1, log, TAG)); + + Integer number = 123_456_789; + Assert.assertSame(number, UtilsInternalLimits.truncateIfString(number, 2, log, TAG)); + Assert.assertEquals("12", UtilsInternalLimits.truncateIfString("123456789", 2, log, TAG)); + } + + /** + * A segmentation is cut and trimmed the same way whatever the order of its map, and never in place. + *

+ * Verifies that a segmentation within every limit, and {@code null}, come back as the same + * instance; that the same entries in two insertion orders give the same result, where of the keys + * cut to the same key the one that sorts last keeps its value and past the entry limit the keys + * that sort first are kept; that the caller's map is left as it was; that a {@code null} key + * sorts first and is never cut; and that cutting values only keeps every key in the order of the map. + */ + @Test + public void segmentation_isCutTheSameWayWhateverTheOrderOfTheMap_andNeverInPlace() { + Map within = TestUtils.map("abc", "value", "number", 12_345_678); + Assert.assertSame(within, UtilsInternalLimits.applySegmentationLimits(within, limits(6, 5, 2), log, TAG)); + Assert.assertNull(UtilsInternalLimits.applySegmentationLimits(null, limits(1, 1, 1), log, TAG)); + Assert.assertEquals(0, warnings.size()); + + Map forward = new LinkedHashMap<>(); + forward.put("colour_a", "red"); + forward.put("colour_b", "green"); + forward.put("colour_c", "blue"); + forward.put("amount", 5); + forward.put("zone", "europe_west"); + Map backward = new LinkedHashMap<>(); + List keys = Arrays.asList(forward.keySet().toArray(new String[0])); + Collections.reverse(keys); + for (String key : keys) { + backward.put(key, forward.get(key)); + } + Map forwardCopy = new HashMap<>(forward); + + Map expected = TestUtils.map("amoun", 5, "colou", "blue"); + Assert.assertEquals(expected, UtilsInternalLimits.applySegmentationLimits(forward, limits(5, 6, 2), log, TAG)); + Assert.assertEquals(expected, UtilsInternalLimits.applySegmentationLimits(backward, limits(5, 6, 2), log, TAG)); + Assert.assertEquals(Arrays.asList("amoun", "colou"), Arrays.asList(UtilsInternalLimits.applySegmentationLimits(backward, limits(5, 6, 2), log, TAG).keySet().toArray())); + Assert.assertEquals(forwardCopy, forward); + + Map withNullKey = new HashMap<>(); + withNullKey.put(null, "null_key_value"); + withNullKey.put("longer_key", 1); + Map limited = UtilsInternalLimits.applySegmentationLimits(withNullKey, limits(3, 100, 1), log, TAG); + Assert.assertEquals(Collections.singletonMap(null, "null_key_value"), limited); + + Map ordered = new LinkedHashMap<>(); + ordered.put("zeta", "long_value"); + ordered.put("alpha", 1); + ordered.put("mid", "short"); + Assert.assertSame(ordered, UtilsInternalLimits.truncateStringValues(ordered, 10, log, TAG)); + Map valuesCut = UtilsInternalLimits.truncateStringValues(ordered, 5, log, TAG); + Assert.assertEquals(Arrays.asList("zeta", "alpha", "mid"), Arrays.asList(valuesCut.keySet().toArray())); + Assert.assertEquals("long_", valuesCut.get("zeta")); + Assert.assertEquals("long_value", ordered.get("zeta")); + } + + /** + * Stack trace lines are cut without touching the line breaks, and the lines of a thread are + * counted from the top. + *

+ * Verifies that a stack trace within the limit comes back as the same instance; that {@code \n} + * and {@code \r\n} breaks, empty lines and a last line without a break are kept, the {@code \r} + * never counting as a character of its line; that one warning is logged per stack trace; and + * that the lines to keep of a thread are the limit only when the thread has more. + */ + @Test + public void stackTraces_linesAreCutWithoutTouchingTheLineBreaks_andLinesPerThreadCountFromTheTop() { + String within = "abc\r\nde\n\nf"; + Assert.assertSame(within, UtilsInternalLimits.truncateStackTraceLines(within, 3, log, TAG)); + + String stackTrace = "java.lang.Exception: message\r\n\tat first.frame(File.java:1)\n\n\tat last.frame(File.java:2)\r\nno_break_at_the_end"; + Assert.assertEquals("java.lang.\r\n\tat first.\n\n\tat last.f\r\nno_break_a", UtilsInternalLimits.truncateStackTraceLines(stackTrace, 10, log, TAG)); + Assert.assertEquals(1, warnings.size()); + Assert.assertTrue(warnings.get(0), warnings.get(0).startsWith(TAG + ", [4] stack trace lines are longer than the line length limit of [10]")); + Assert.assertEquals("12\r\n34\r\n", UtilsInternalLimits.truncateStackTraceLines("123\r\n345\r\n", 2, log, TAG)); + + Assert.assertEquals(5, UtilsInternalLimits.stackTraceLinesToKeep(5, 5, "worker", log, TAG)); + Assert.assertEquals(2, warnings.size()); + Assert.assertEquals(3, UtilsInternalLimits.stackTraceLinesToKeep(5, 3, "worker", log, TAG)); + Assert.assertTrue(warnings.get(2), warnings.get(2).startsWith(TAG + ", the stack trace of the thread [worker] has [5] lines, over the limit of [3]")); + } + + /** + * A settings provider serving the given key length, value size and segmentation entry limits. + * + * @param maxKeyLength the key length limit + * @param maxValueSize the value size limit + * @param maxSegmentationValues the segmentation entry limit + * @return the provider + */ + private static ConfigurationProvider limits(int maxKeyLength, int maxValueSize, int maxSegmentationValues) { + ModuleConfiguration limits = new ModuleConfiguration(); + limits.currentVMaxKeyLength = maxKeyLength; + limits.currentVMaxValueSize = maxValueSize; + limits.currentVMaxSegmentationValues = maxSegmentationValues; + return limits; + } +}