Endless scroll not requesting another page

[Brian]

I'm trying to implement the endless scroll.  When I just get all my data, all works.  When I put in the endlessScroll and scrollThreshold for the list view, parameterMap, pageSize, and serverPaging for the data source, I only get the first page of 10.  I get the url request: /api/company?take=10&skip=0&page=1&pageSize=10 for the first page and that works. But when I want more data appended to the bottom of the list by dragging the list up so I see the 10th record on the bottom, another request is not made.  What am I missing for Kendo to make that next request for the next page?

  var companyDataTransport = new kendo.data.RemoteTransport({
      read: {
          url: base_url + "/company",
          dataType: "json",
          type: "GET"
      },
  });

  var dsCompany = new kendo.data.DataSource({
      pageSize: 10,
      serverPaging: true,
      schema: {
        type: "json",
        model: Company
       },
   transport: companyDataTransport,
   parameterMap: function (options) {
       var parameters = {
           max: options.pageSize,
           page: options.page
       }
       return parameters;
   },

  });

  function mobileServicersDataBind() {
   $("#servicerView").kendoMobileListView({
    dataSource: dsCompany,
    template: kendo.template($('#servicerOnOffTemplate').html()),
    endlessScroll: true,
    scrollThreshold: 30
   });
  };

[Alexander Valchev]

Hi Brian,

Thank you for getting in touch with us.

I assume that the problem is connected with the total property of the DataSource. Please check this forum post for possible solutions.
In addition, there is a syntax mistake in the DataSource configuration - parameterMap function should be part of the transport object.

Regards,
Alexander Valchev
the Telerik team

[Brian]

Thanks for that link and help.  With the schema.total set and removing my <li>'s from the template, the endless scroll worked.  It made the url requests to the server as I expected.  However, when I try to implement the endless scroll with my production template, I have issues.  The template does not render properly for the subsequent pages of data and I can't update from the list item switch after a scroll.  Template is servicerOnOffTemplate below and works fine when I get all data at once.  When I implement the endless scroll and get the first page of data, it renders and looks fine.  When I scroll and it loads the next page, my company names render but the switch does not.  What is wrong with my template that I get the names but not the switch for pages 2 and beyond?  On the first page where I have the switches, clicking/sliding the switch calls the function and gets the record if I have not scrolled, and the update works.  But if I have scrolled and loaded a second page, scroll back and activate one of the switches from the first page where they did render, it does not get the record, there is some sort of failure with the get on the data source.  Does the endless scroll not append that subsequent data into the data source so all records in the list view are available? I think I have all the relevant code below.  Again, if I remove the pageSize, serverPaging, and schema.total from the data source and the endlessScroll from the list view, it all works.  Let me know what to try.

var companyDataTransport = new kendo.data.RemoteTransport({

    read: {

        url: base_url + "/company",

        dataType: "json",

        parameterMap: function (options, type) {

            var parameters = {

                take: options.take,

                skip: options.skip,

                pageSize: options.pageSize

            }

            return parameters;

        },

        type: "GET"

    },

    update: {

        url: base_url + "/company",

        dataType: "json",

        data: {

            Company: function () {

                return JSON.stringify(editComp);

            }

        },

        type: "POST"

    }

});

 

var dsCompany = new kendo.data.DataSource({

    pageSize: 10,

    serverPaging: true,

    schema: {

        type: "json",

        model: Company,

        total: function () {

            return totalServicerCount;

        }

    },

    transport: companyDataTransport,

    error: function (e) {

        alert("dsCompany error: " + e.status + " errorThrown: " + e.errorThrown);

    }

});

 

<div id="servicerOnOff" data-role="view" data-title="Turn Servicer On/Off" data-init="mobileServicersDataBind" style="display:none">

    <ul data-role="listview" data-style="inset">

        <ul id="servicerView"></ul>

    </ul>

</div>

 

function mobileServicersDataBind() {

    $("#servicerView").kendoMobileListView({

        dataSource: dsCompany,

        template: kendo.template($('#servicerOnOffTemplate').html()),

        endlessScroll: true

    });

};

 

<script type="text/x-kendo-template" id="servicerOnOffTemplate">

    #

    if (IsOn)

    {

        #

            <input type="checkbox" data-customvalue="#:CompanyID#" data-role="switch" checked="checked" data-change="switchServicerOnOff"/> #= Name #

        #

    }

    else

    {

        #

            <input type="checkbox" data-customvalue="#:CompanyID#" data-role="switch" data-change="switchServicerOnOff"/> #= Name #

        #

    }

    #

</script>

 

function switchServicerOnOff(e) {
    editComp = dsCompany.get(e.sender.element.context.dataset.customvalue);
    editComp.set("IsOn", e.checked);
    dsCompany.sync();
};

[Alexander Valchev]

Hello Brian,

Does the endless scroll not append that subsequent data into the data source so all records in the list view are available?

When serverPaging is enabled, the DataSource keeps only the dataItems which are part the current page. Previous records are not available any more which is why retrieving the dataSource records fails.
I am afraid that this behaviour is by design and cannot be prevented.

The only solution that I can offer is to store information about the record in the mark-up via data attributes and update it through custom Ajax request.

Please accept my apology for the inconvenience caused.

Regards,
Alexander Valchev
the Telerik team

[Brian]

While it is confusing that the data is not available in the data source even though the list item shows the data, I can accept that this is they way it is.  This does not answer why the subsequent pages of the list do not render the template correctly with the switches.  I tested that with the plainest of templates with only the code below and it still does not render correctly for the second page and beyond.  First page is fine, but when the second page is loaded, there is nothing but empty list lines. If I include #= Name # after the input markup, it shows my company names for all pages, but not the switch.  Is this something I should open a service ticket for?

<script type="text/x-kendo-template" id="servicerOnOffTemplate"> 
    <input type="checkbox" data-role="switch" />
</script> 

[Alexander Valchev]

Hello Brian,

I am not sure exactly where the problem comes from. I tried to reproduce the behaviour here but to no avail.
Could you please check my sample and let me know what I am missing? On my side the switch widget renders for both subsequent and appended on top records. What KendoUI version are you using? If it is an outdated one, please download the Service Pack release (2012.3.1315) and test your project with it.

Can you reproduce the issue in a jsBin and send me back a link so I can check what is going wrong? Alternatively you may open a support ticket with small but runnable project attached.
Thank you in advance for the cooperation.

Kind regards,
Alexander Valchev
the Telerik team

[Brian]

I'm running version 2012.3.1114, and I can't find the service pack you reference on your site.  Your sample uses a pull to refresh model so your new data and newly rendered switch goes at the top of the list.  I have not found a sample or demo on your site yet that uses server paging and does NOT use Twitter.  My list of data is essentially an alphabetical list where the first call to the server loads 20 records, A-D, the next call to the server would load another 20, D-G, and so on.  I don't have time to work up a sample and we've moved on anyway to just getting the data in alpha batches instead of server paging.  I'm convinced your tool is broken with that switch not rendering on subsequent pages that load at the botton, not top like your sample.  Kendo's implementation of server paging was bad anyway with the items visible on the list but not available in the data source after scrolling.  I do thank you for your time though.

[Alexander Valchev]

Hi Brian,

To download the service pack please log in with your account at www.kendoui.com and go to Your Account > Your Products > Download List. Service Pack release is the default version for download. For your convenience I attached a screen shot.

Since you did not specified which KendoUI version you used I tested only with the latest public release. Indeed the problem occurs with v2012.3.1114 however I did not manage to reproduce it with the Service Pack release.

Actually the example that I provided in my previous post uses both endless scrolling and pull-to-refresh. Here is a version that uses only endless scroll. Can you reproduce the issue in this jsBin? If not could you please open a support ticket with a sample project attached so I can check what is going wrong?

Kind regards,
Alexander Valchev
the Telerik team

[Brian]

I did specify which version I was using, first line of my last post.  I worked up the test project that exhibited the problem using that 2012.3.1114 version.  But before sending to you I would try the service pack upgrade.  I eventually did find the sevice pack install, installed it, manually copied all the css and js files referenced into my test project.  (Is there not an automatic upgrade for an existing solution, or do I need to create the 2012.3.1315 folders and manually copy the js and css files in?)  And yes, you have fixed the problem in this release.  The switches now render for subsequent pages.  Thanks for checking it out and getting me to the service pack upgrade.

edit: I found the Upgrade wizard on the Telerik menu.  Thanks again for all the help.

[Kevin]

When serverPaging is enabled, the DataSource keeps only the dataItems which are part the current page. Previous records are not available any more which is why retrieving the dataSource records fails. I am afraid that this behaviour is by design and cannot be prevented.

This is extremely limiting, for no good reason. It is fine as "default" functionality, but there should be an option to cause the dataSource to retain data for everything that is displaying. This is necessary to make it useful for "lazy loading". A key reason for using data caching and templates is to avoid re-downloading data repeatedly. That is a waste of bandwidth.

I have a listView with functionality to view a 'detail screen', like the Sushi sample app. However unless I download ALL the data at once - and have poor performance as a result - I end up with a list where detail functionality is 'broken' on all but the last page of data retrieved. That's crazy! If the data is displays on the page, then it should exist in the dataSource. That's what data objects are for!

Copying all the item data as data-attributes is a grossly inadequate solution, which also will not work with detail-screen data-binding and other dataSource/viewModel functionality. So using such a hack would require writing code to replicate the required functionality - not practical.

If the goal is to maximize performance, then a far more useful process would be to remove listView items as users scroll past them, and regenerating them if they scroll-up again. This is how many sophisticated lazy-loading widgets work. Keeping the DOM small when working with big lists, (especially on mobile devices), would provide much more benefit than removing dataSource array data.

In my opinion an option should be added to allow the dataSource to work for normal lazy-loading. The capability to add and retain data already exists, so this should take very little effort. It would address important issues encountered by many Kendo users, like the other people in this thread, and me.

Instead I will have to hack the dataSource to make it work. If I can't easily modify its paging behavior, then I'll cache the existing data, add the newly fetched data, then push the combined data back into the dataSource each page. Unfortunately this means the whole listView will also have to regenerate for each page. That is an ugly hack that will hurt performance and usability. A simple option to 'retain data' would solve my problem.

Kendo is a great set of tools, but some things require more versatility to make them suitable for advanced UI designs with complex functionality. This is one of those things, and a very important one because the dataSource is at the heart of everything.

Cheers

[Alexander Valchev]

Hi Kevin,

Thank you for the feedback.

We agree that the current implementation has drawbacks, especially in scenarios like the one you described. The main obstacle for us to change the behaviour is the complexity of its implementation on a DataSource level.
We will research the possible opportunities and try to provide a solution in one of our future releases. I am afraid that I cannot commit an exact time frame when this will happen.
Please accept my apology for the inconvenience caused.

Kind regards,
Alexander Valchev
the Telerik team

[Kevin]

Hi Alexander,

Thanks for replying. I realize this is not necessarily a quick-fix, but it would be good to make it a goal so that all updates can keep this in mind.

I am trying to use the dataSource as a multipurpose client-side object, because it doesn't make sense to store the same data in multiple places. I'm also using the listView in ways beyond simple fetch-and-display. These two widgets are so closely linked that the listView is like an extension to the dataSource.

I'm still tweaking this code, but it seems I'll be able to get the basics I need without many changes. So far small changes to the dataSource.success() and listView.refresh() methods are giving me a lot of flexibility. All I'm doing is passing in an options param into the success() method, which then sets those options in the 'e' param it passes to the _process() method, which then these to all listeners, like a listView. None of this breaks existing functionality because things only change is you pass options nto the success() method from the ds.transport.read() function. Or these options could be specified in ds.options for users who don't want to use a custom read function.

Extending the current ds functionality won't be that hard, and doesn't necessitate changes across all widgets (like Grid). It has only been a pain for me because I had to reverse-engineer everything to understand how the flow works. The dataSource part is easy - the listView is actually the harder part!

When I have all the bugs worked out, I'll post my enhancements here as a starting point for possible ds enhancements. If nothing else it may be useful to others, who in turn may improve it and post those changes back too.

-----

IN THE MEANTIME, here are SOME OTHER extensions I added to suit my needs. I'll pass these on in hopes that some can make it into a future Kendo release, in one form or another...

The listView has a stopEndlessScrolling method, but no 'start' method! I need this for my use, so I added one. In this case it is a 'restart' method because I start with endlessScrolling enabled, but it could be easily updated to start (bind) when the option was not originally enabled. I added an options param too, but that is secondary to its main purpose...

$.extend( true, kendo.mobile.ui.ListView.prototype, {

    // re-enable endlessScroll after stopEndlessScrolling stopped it
    restartEndlessScrolling: function ( options ) {
        var that    = this
        ,    scroller= that._scrollerInstance
        ,    o        = options || {}
        ;
        // only reset if currently not currently enabled
        if ( !that._scrollerInstance._events.resize.length ) {
            scroller.setOptions({
                resize: that._scrollerResize
            ,    scroll: that._scrollerScroll
            });
        }
        // handle options
        if ( o.scrollTop )
            scroller.scrollTo( 0, -o.scrollTop ); // number must be negative
        if ( o.nextPage && !that.loading ) // ensure was not triggered by scrollTo above
            that._nextPage.call( that );
    }

});

I'm using websockets, so I use s ds.transport.read function as a pass-through for my websocket-transport API. I also added transport.push && transport.load methods that I use as listeners for events I trigger when data is 'pushed' through the websocket. These simple methods then use the ds load, add or append methods, as appropriate.

Because I receive BOTH 'new items' and 'updates' (like status-changes) to items I've already stored in the ds, I added methods to the dataSource to handle it without having to repeat the code everywhere I need it...

$.extend( true, kendo.data.DataSource.prototype, {

     getOrAdd: function ( data, prepend ) {
        var    that    = this
        ,    curItem    = that.get( data[ that.options.schema.model.idField ] || 0 )
        ;
        if ( curItem )
            return curItem;

        // PARSE the new data - uses ds.schema.parse
        data = that.reader.parse( data );

        // add the new item and return it
        return prepend ? that.insert( 0, data ) : that.add( data );
    }

,    addOrUpdate: function ( data, prepend ) {
        var    that    = this
        ,    curItem    = that.get( data[ that.options.schema.model.idField ] || 0 )
        ;
        // PARSE the new data - uses ds.schema.parse
        data = that.reader.parse( data );

        // update the current item, if exists
        if ( curItem )
            return that.update( curItem, data );

        // add a new item and return it
        return prepend ? that.insert( 0, data ) : that.add( data );
    }

,    update: function ( item, newData ) {
         var e = {
            action:    'update'
        ,    items:    [ item ]
        ,    fields:    [] // array of all 'changed fields'
        ,    values:    {} // hash of all 'changed fields/values
        };
        go( newData, '' ); // process field/branch
        // TODO: should we update the ds._pristine data too???

       item.dirty = false; // updated from the server, so item is now in-sync - NOT dirty

        e.changed = !!e.fields.length;

        if ( e.changed )
            this.trigger('change', e );

        // recursive parsing function
        function go ( branch, path ) {
            var key, keyPath, newVal;
            // TODO: handle keys containing an array of values - simple or hash
            for ( key in branch ) {
                newVal    = branch[key];
                keyPath    = (path ? path +'.' : '') + key;
                if ($.isPlainObject( newVal ))
                    go( newVal, keyPath );
                else {
                    d = wk.util.getObjectBranch( item, keyPath );
                    if ( d.key && d.value !== newVal ) {
                        d.branch[ d.key ] = newVal;
                        e.fields.push( keyPath );
                        e.values[ keyPath ] = newVal;
                    }
                }
            }
        };
    }

});