Dear Community,
We have tried to run a Multi-armed...
# ask-questionsb
busy-cat-1027
03/10/2025, 3:48 PMDear Community,
We have tried to run a Multi-armed bandit test recently, but bumped into so many obstacles...
First: the exploratory Stage was 1 day, but I had to manually finish it after 5 days because it did not start automatically. The update Cadence was set to every 1 days but it did not work either. You can see it on the result diagram.
We got the error notices as well about the Sample Ratio Mismatch (SRM) (P-value below 0.001) and about the Multiple Exposures (102,323 users (38.47%) saw multiple variations and were automatically removed from results).
And as you can see, somehow the starting date of the test is the date when we create the bandit, not the start. We have no idea why.
Can anyone explain these? Because we have no idea how to solve the problem.
s
strong-mouse-55694
03/10/2025, 6:18 PMSorry! That doesn't look right. Can you share a screenshot of the experiment setup? And how did you implement sticky bucketing?
h
helpful-application-7107
03/10/2025, 6:18 PM(and are you using mixpanel?)
b
busy-cat-1027
03/10/2025, 7:04 PMHere is the sticky bucketing implementation:
busy-cat-1027
03/10/2025, 7:06 PMAnd I have no idea what to show when you ask about the experiment setup. Our variations are only visual changes.
s
strong-mouse-55694
03/10/2025, 9:49 PMThanks for sharing that. Can you share a bit more about how you're implementing the SDK? It looks a bit different than our docs. That doesn't mean it won't work, but we need some more info on it.
b
busy-cat-1027
03/11/2025, 7:55 AMWell, yes, but it is exactly as in the docs (as you can see in the second line of the sdk script)
busy-cat-1027
03/11/2025, 7:57 AM<script id="growthbook-sdk" src="https://cdn.jsdelivr.net/npm/@growthbook/growthbook/dist/bundles/index.min.js" defer></script>
<script data-use-sticky-bucket-service="cookie">
var isMobile = false;
(function(a){if(/(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(a)||/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test(a.substr(0,4))) isMobile = true;})(navigator.userAgent||navigator.vendor||window.opera);
(function() {
// Wait for the SDK to load before starting GrowthBook
var clientId = {{gtagApiResult.client_id}};
if (window.growthbook) {
startGrowthbook();
} else {
document.querySelector("#growthbook-sdk").addEventListener("load", startGrowthbook);
}
function startGrowthbook() {
if (!window.growthbook) return;
var gb = new growthbook.GrowthBook({
apiHost: "https://cdn.growthbook.io",
clientKey: "xxxxxx",
enableDevMode: true,
// TODO: Add decryptionKey if using encryption
attributes: {
id: clientId, // TODO: Read user/device id from a cookie/datalayer
user_id: {{dlv - businessPartnerId}},
mobile: isMobile,
loggedIn: {{cJS - logged_in}},
user_has_subscription: {{dlv - userHasSubscription}},
user_logged_in_type: {{GA4 - userTypeJS}},
listing_type: {{dlv - listingType}},
property_type: {{dlv - propertyType}},
content_group: {{dlv - pageType}}
},
trackingCallback: function(experiment, result) {
// TODO: track experiment impression
dataLayer.push({
"event": "experiment_viewed",
"experiment_id": experiment.key,
"variation_id": result.variationId
});
}
});
gb.loadFeatures({ autoRefresh: true });
// TODO: Instrument DOM with AB test logic
}
})();
</script>
s
strong-mouse-55694
03/11/2025, 2:42 PMBecause you have the
data-use-sticky-bucket-service="cookie"b
busy-cat-1027
03/11/2025, 3:49 PMOhh, so we should rather put that into the first script tag?
busy-cat-1027
03/11/2025, 3:50 PMAnd yes, you can check the site here: https://ingatlan.com
busy-cat-1027
03/14/2025, 12:43 PMHey Community,
I still have these questions open...
s
strong-mouse-55694
03/14/2025, 2:17 PMHi @busy-cat-1027, yes, please check out the examples from our docs: https://docs.growthbook.io/lib/script-tag#adding-custom-attributes. I think it'd be good to follow the example there as a baseline for troubleshooting, as it's a configuration/set up that is known to work.
That being said, I didn't the script at all on the page you shared. (of course, I could've missed it 😅)
16 Views