( ! ) Deprecated: strip_tags(): Passing null to parameter #1 ($string) of type string is deprecated in /home/jlahijani/Sites/goodui.org/site/templates/_og.php on line 13
| Call Stack |
|---|
| # | Time | Memory | Function | Location |
|---|
| 1 | 0.0001 | 465800 | {main}( ) | .../index.php:0 |
| 2 | 0.2635 | 2254824 | ProcessWire\Wire->__call( $method = 'execute', $arguments = [0 => TRUE] ) | .../index.php:55 |
| 3 | 0.2635 | 2254824 | ProcessWire\WireHooks->runHooks( $object = class ProcessWire\ProcessPageView { }, $method = 'execute', $arguments = [0 => TRUE], $type = ??? ) | .../Wire.php:484 |
| 4 | 0.2635 | 2255240 | ProcessWire\Wire->_callMethod( $method = '___execute', $arguments = [0 => TRUE] ) | .../WireHooks.php:952 |
| 5 | 0.2635 | 2255240 | ProcessWire\ProcessPageView->___execute( $internal = TRUE ) | .../Wire.php:416 |
| 6 | 0.2690 | 2295640 | ProcessWire\ProcessPageView->renderPage( $page = class ProcessWire\Page { public $id = 3170; public $name = 'simulating-how-long-i-should-run-my-test'; public $parent = '/blog/'; public $template = 'post'; public $title = 'Simulating How Long To Run Your Test'; public $post_tags = '(PageArray) 3143'; public $body_blocks = '(RepeaterMatrixPageArray) 6581'; public $date = 1461098916; public $comments = '(CommentArray) 472|471'; public $summary = ''; public $body = ''; public $data = ['title' => 'Simulating How Long To Run Your Test', 'post_tags' => class ProcessWire\PageArray { ... }, 'body_blocks' => class ProcessWire\RepeaterMatrixPageArray { ... }, 'date' => 1461098916, 'comments' => class ProcessWire\CommentArray { ... }, 'summary' => '', 'body' => ''] }, $request = class ProcessWire\PagesRequest { } ) | .../ProcessPageView.module:114 |
| 7 | 0.2794 | 2325776 | ProcessWire\Wire->__call( $method = 'render', $arguments = [] ) | .../ProcessPageView.module:184 |
| 8 | 0.2794 | 2325776 | ProcessWire\WireHooks->runHooks( $object = class ProcessWire\Page { public $id = 3170; public $name = 'simulating-how-long-i-should-run-my-test'; public $parent = '/blog/'; public $template = 'post'; public $title = 'Simulating How Long To Run Your Test'; public $post_tags = '(PageArray) 3143'; public $body_blocks = '(RepeaterMatrixPageArray) 6581'; public $date = 1461098916; public $comments = '(CommentArray) 472|471'; public $summary = ''; public $body = ''; public $data = ['title' => 'Simulating How Long To Run Your Test', 'post_tags' => class ProcessWire\PageArray { ... }, 'body_blocks' => class ProcessWire\RepeaterMatrixPageArray { ... }, 'date' => 1461098916, 'comments' => class ProcessWire\CommentArray { ... }, 'summary' => '', 'body' => ''] }, $method = 'render', $arguments = [], $type = ??? ) | .../Wire.php:484 |
| 9 | 0.2794 | 2327736 | ProcessWire\Wire->__call( $method = 'renderPage', $arguments = [0 => class ProcessWire\HookEvent { public $data = [...] }] ) | .../WireHooks.php:1060 |
| 10 | 0.2794 | 2327736 | ProcessWire\WireHooks->runHooks( $object = class ProcessWire\PageRender { public $data = ['clearCache' => 1] }, $method = 'renderPage', $arguments = [0 => class ProcessWire\HookEvent { public $data = [...] }], $type = ??? ) | .../Wire.php:484 |
| 11 | 0.2795 | 2328152 | ProcessWire\Wire->_callMethod( $method = '___renderPage', $arguments = [0 => class ProcessWire\HookEvent { public $data = [...] }] ) | .../WireHooks.php:952 |
| 12 | 0.2795 | 2328152 | ProcessWire\PageRender->___renderPage( $event = class ProcessWire\HookEvent { public $data = ['object' => class ProcessWire\Page { ... }, 'method' => 'render', 'arguments' => [...], 'return' => NULL, 'replace' => FALSE, 'options' => [...], 'id' => 'Page:100.0:render', 'cancelHooks' => FALSE, 'when' => 'after'] } ) | .../Wire.php:416 |
| 13 | 0.2801 | 2328704 | ProcessWire\Wire->__call( $method = 'render', $arguments = [] ) | .../PageRender.module:575 |
| 14 | 0.2801 | 2328704 | ProcessWire\WireHooks->runHooks( $object = class ProcessWire\TemplateFile { public $data = ['wire' => class ProcessWire\ProcessWire { ... }, 'classLoader' => class ProcessWire\WireClassLoader { ... }, 'hooks' => class ProcessWire\WireHooks { ... }, 'config' => class ProcessWire\Config { ... }, 'notices' => class ProcessWire\Notices { ... }, 'urls' => class ProcessWire\Paths { ... }, 'log' => class ProcessWire\WireLog { ... }, 'sanitizer' => class ProcessWire\Sanitizer { ... }, 'datetime' => class ProcessWire\WireDateTime { ... }, 'files' => class ProcessWire\WireFileTools { ... }, 'mail' => class ProcessWire\WireMailTools { ... }, 'database' => class ProcessWire\WireDatabasePDO { ... }, 'db' => class ProcessWire\DatabaseMysqli { ... }, 'cache' => class ProcessWire\WireCache { ... }, 'modules' => class ProcessWire\Modules { ... }, 'procache' => class ProcessWire\ProCache { ... }, 'fieldtypes' => class ProcessWire\Fieldtypes { ... }, 'fields' => class ProcessWire\Fields { ... }, 'fieldgroups' => class ProcessWire\Fieldgroups { ... }, 'templates' => class ProcessWire\Templates { ... }, 'pages' => class ProcessWire\Pages { ... }, 'permissions' => class ProcessWire\Permissions { ... }, 'roles' => class ProcessWire\Roles { ... }, 'users' => class ProcessWire\Users { ... }, 'user' => class ProcessWire\User { ... }, 'session' => class ProcessWire\Session { ... }, 'input' => class ProcessWire\WireInput { ... }, 'buster' => class ProcessWire\ProCacheBuster { ... }, 'process' => class ProcessWire\ProcessPageView { ... }, 'page' => class ProcessWire\Page { ... }, 'options' => [...]] }, $method = 'render', $arguments = [], $type = ??? ) | .../Wire.php:484 |
| 15 | 0.2801 | 2330056 | ProcessWire\Wire->_callMethod( $method = '___render', $arguments = [] ) | .../WireHooks.php:952 |
| 16 | 0.2801 | 2330056 | ProcessWire\TemplateFile->___render( ) | .../Wire.php:413 |
| 17 | 0.3917 | 3024472 | require( '/home/jlahijani/Sites/goodui.org/site/templates/_main.php ) | .../TemplateFile.php:340 |
| 18 | 0.3929 | 3067784 | include( '/home/jlahijani/Sites/goodui.org/site/templates/_og.php ) | .../_main.php:64 |
| 19 | 0.3949 | 3068520 | strip_tags( $string = NULL ) | .../_og.php:13 |
" />
Simulating How Long To Run Your Test
Tagged as:
Optimization
How much time is enough for the true performance of your variations to come through the noise?
In this video, we'll see a simulation of an A/A/B/C/D test as it moves from the initial state dominated by chance towards a state of equilibrium. In the process, we observe how the performance of variations can change over time due to chance alone and what sorts of intermediate outcomes we can expect. How does a false positive tend to behave over time? What is a true +10% winner likely to do half way into the test? Answering these questions helps me interpret real tests.
To speed things up, this simulation is based on a 20% baseline conversion rate and 1,000 visitor hits per day. The duration of 10 days is just an example. In your real tests, the conversion rate might be as low as 1%, which means it would take far longer to get to a similar equilibrium.
Exercise:
- Use Evan Miller's Sample Size Calculator to calculate the sample size needed to detect a 10% relative lift over a 20% baseline (answer is at the bottom of this post) - leave the power and significance on default.
- Rewatch the simulation video and see how the test behaves as it approaches this sample size target.
- Consider: How accurate is the relative performance of each variation at this point? What sort of outcomes are still possible by chance alone that would obscure the true performance of the variations? Based on this simulation would you run your test longer or less than this target?
Do you use simulations for planning and analysis? Share with us.
(Answer: 6347 visitors per variation)
Posted by 
Jakub Linowski
on Apr 19, 2016
Reach Higher Conversions Faster With These Actively Tested Patterns:
Comments
Gavin Morrice 9 years ago ↑1↓0
I just built a small Ruby program that simulates this, the results are pretty mind-blowing.
It takes several thousand samples before you get an accurate result with Split A/B testing. Even more if you're using something like an Epsilon Greedy algorithm.
Have a play around with it, the code is on GitHub
https://github.com/Bodacious/OptimisationTestComparisons/
Reply
Jakub Linowski 9 years ago ↑0↓0
Hey Gavin. Awesomeness. These simulations are nice in that they visualize false positives and false negatives. Agreed. And yes, it usually takes thousands of visitors for the patterns to stabilize. I think the key factors are: sample size, baseline conversion rate, and the magnitude of effect.
Unfortunately, I don't know Ruby to run your script :(
Cheers,
J
Reply