a8m
Performance issue in groupBy
I have an array of 300 non-trivial objects, which I want to filter with groupBy. When I attempt to do that, my whole applications slows to a crawl.
This is due to enormous amounts of time being spent in getHashKey, which JSON.stringifys the whole array to use that as the key for $$cache.
If there's an alternative approach, I'd love to hear it :)
We moved to a solution that works for our specific use case. It relies on the objects being grouped having an id property:
angular
.module( "util" )
.filter( "groupBy", groupByFilterProvider );
/* @ngInject */
function groupByFilterProvider( _ ) {
return function groupByFilter( array, groupKey ) {
if( !array || !array.length ) {
return array;
}
if( array.__groupByContainer ) {
var arrayId = idString( array, groupKey );
if( array.__groupById === arrayId ) {
return array.__groupByContainer;
}
}
array.__groupByContainer = {};
array.__groupById = idString( array, groupKey );
var elementProperty;
_.forEach( array, function addElementToGroup( element ) {
elementProperty = element[ groupKey ];
if( !array.__groupByContainer[ elementProperty ] ) {
array.__groupByContainer[ elementProperty ] = [];
}
array.__groupByContainer[ elementProperty ].push( element );
} );
return array.__groupByContainer;
};
function idString( array, key ) {
return _.reduce( array, function appendKey( id, element ) {
return id + element.id + element[ key ];
}, "" );
}
}
+1 for this issue. groupBy works great for simple object arrays, but with complex objects it is really slow. The hashKey that's generated for a simple demo array of 62 complex objects is 61Kb. In the actual application, the array will be much larger.
Update - I took a similar-ish approach to @oliversalzburg above. The hashKey I'm creating for the group objects is a concatenation of the object keys (as opposed to a JSON of everything like the actual library). So I call _groupBy with each iteration to get the keys, then pass them to a modified version of isMemoized that uses those keys instead of the giant one that it is currently using. Simply changing the keys has taken a 10 second plus render cycle to under 1 second...and calling _groupBy with each iteration only takes a millisecond or two.
Hi @oliversalzburg ! On your code, what is the meaning of the underscore groupByFilterProvider( _ )?
I just pasted your code and it throws this error Unknown provider: _Provider <- _ <- groupByFilter.
Edited:
I removed the _ from the function parameter and added lodash to make it work.
We usually wrap globals in Angular providers so that we can mock them. A simple way would be angular.module("foo").constant("_",_);
I changed my approach from my previous post. Now, I'm creating a unique key by using the grouping property concatenated with a random number. Then when the cache is checked, it looks for the non-random part of the key and checks to see if the collection is the same (the random part of the key is inferred from the collection match). So...
.filter('groupBy', [ '$rootScope', '$parse', 'filterWatcher', function ( $rootScope, $parse, filterWatcher ) {
return function (collection, groupOn) {
if(!isObject(collection) || isUndefined(groupOn) || !collection.length || collection.length === 0) {
return collection;
}
var key = filterWatcher.getKey(collection, groupOn),
grpObj = _groupBy(collection, $parse(groupOn));
return filterWatcher.isMemoized(arguments, key) ||
filterWatcher.memoize(arguments, key, grpObj);
/**
* groupBy function
* @param collection {Array}
* @param key {String}
* @returns {{}}
*/
function _groupBy(collection, key) {
var result = {},
prop = '';
forEach(collection, function(item, i) {
prop = key(item);
if(prop == '0' || prop == ''){
prop = 'zz'+ prop;
}
if(!result[prop]) {
result[prop] = [];
}
result[prop].push(item);
});
return result;
}
}
}]);
...And then in the filterWatcher, I changed and added...
/**
* @private
* isSame
* - Checks to see if the objects in the array are the same
* @param arr1 {Array}
* @param arr2 {Array}
* @returns {Boolean}
*/
function isSame(arr1, arr2){
if(arr1.length !== arr2.length){
return false;
}
for(var i=0; i<arr1.length; i++){
if(arr1[i] !== arr2[i]){
return false;
}
}
return true;
}
/**
* @description
* return the `cacheKey` or undefined.
* @param args {Array}
* @param key {String} // GHY addition!
* @returns {*}
*/
function $$isMemoized(args, key) {
var cache = $$cache[key];
if(!cache || (cache && !isSame(cache.$$collection, args[0]))){
return null;
}
return cache.$$result;
}
/**
* @description
* store `result` in `$$cache` container, based on the hashKey.
* add $destroy listener and return result
* @param args {Array}
* @param key {String}
* @param result {Object}
* @returns {Object}
*/
function $$memoize(args, key, result) {
$$cache[key] = {
$$result: result,
$$collection: args[0]
};
cleanStateless();
return result;
}
/**
* @private
* getKey
* - Returns a new key if the array is new
* @param arr {Array}
* @param groupOn {String}
* @returns {String}
*/
function $$getKey(arr, groupOn){
var key = '';
for(key in $$cache){
if(key.indexOf(groupOn) === 0 && isSame($$cache[key].$$collection, arr)){
return key;
}
}
return newKey(groupOn);
}
/**
* @private
* newKey
* - Ensures a unique key is generated
* @param groupOn {String}
* @returns {String}
*/
function newKey(groupOn){
var key = groupOn +'_'+ Math.random().toString().substring(2);
if($$cache[key]){
return newKey();
}
return key;
}
...Checking the object pointers within the array (the isSame function) is much quicker than matching on massive object keys (at least in my case).
I was also having heavy performance issue. Sad that this filter is so bad optimized. For me I'm now creating an array of "groups" then on the ng-repeat I only show that specified item if it belongs to that repeated group. Believe or not this works Much faster than this filter... In any case I'm gonna improve my code.
<div ng-repeat="(groupKey, value) in groups track by $index">
<div class="item item-divider divider">
{{groupKey}}
</div>
<div class="item item-text-wrap" ng-repeat="item in menu" ng-if="item.CAT_DESCRIPTION==group">
<h2>{{item.name}}</h2>
<div class="row"> <div class="col-80"> <p>{{item.description}}</p> </div> <div class="col-20" > {{item.price}} </div> </div> </div> </div>
