Custom Taxonomy terms with latest post ordered by date pagination issue

I’m using the similar code from this question, order terms by post date works only if I display all the terms on the page, example terms_to_display = 99999, but if I set it to 6 or for example, the complete loop order gets messed up, it’s not continuing on the next pages, it starts again from the newest post again ( minus offset of 6 ) and displays the next 6 posts by date, resulting with having newer posts on the second,third… pages than on the first.

    $taxonomy = 'ctc_sermon_series'; //taxonomy name
    $terms_to_display = 6; //number of series to display per page
    $page         = ( get_query_var( 'paged' ) ) ? get_query_var( 'paged' ) : 1;
    $totalterms   = wp_count_terms( $taxonomy, array( 'hide_empty' => TRUE ) ); 
    $totalpages   = ceil( $totalterms / $number );
    $posts_per_term = 1; // number of posts per term to display

    $term_count = get_terms( array (
        'taxonomy' => $taxonomy,
        'fields'   => 'count',
     ) );

    // find out the number of pages to use in pagination
    $max_num_pages = ceil( $term_count / $terms_to_display );

    // get the page number from URL query
    $current_page = get_query_var( 'paged', 1 );

    // calculate the offset, if there is one.
    $offset = 0; // initial
    // or changed the if not the first (0)
    if( ! 0 == $current_page) {
        $offset = ( $terms_to_display * $current_page ) - $terms_to_display;
    }


    $terms = get_terms ( array (
        'taxonomy'      => $taxonomy, 
        'hide_empty'    => true,
        'number'        => $terms_to_display,
        'offset'        => $offset,
        ) 
    );
    $sermons_items = [];

    foreach ( $terms as $term ) {

        $args = array (
            'post_type'         => 'ctc_sermon',
            'posts_per_page'    =>  $posts_per_term,
            'orderby'           => 'date',
            'order'             => 'DESC',
            'tax_query' => array (
             array (
                'taxonomy' => $taxonomy,
                'field'    => 'id',
                'terms'    => array( $term->term_id )
                ),
             ),
          );

        $sermons_query = new WP_Query( $args );

        // ignore terms with not enough posts
        if ( $sermons_query->post_count < $posts_per_term ) {
            continue;
        }

        $sermons_posts = $sermons_query->get_posts();

        // get date of newest post in this term
        $newest_post_date = get_the_date( "U", $sermons_posts[0]->ID );

        $sermons_items[$newest_post_date] = array(
            'term' => $term,
            'sermons' => $sermons_posts,
        );
    }

    krsort( $sermons_items ); // sort descending by keys

    $sermons_items = array_slice( $sermons_items, 0, $terms_to_display );


    global $post;

    echo '<section class="grid-posts">';

    foreach ( $sermons_items as $item ) {

        echo '<div class="col-md-6 no-padding grid-post">'; 

        foreach ( $item['sermons'] as $post ) {
            setup_postdata( $post ); ?>

            <div class="grid-post-content">
                <a href="<?php echo esc_url( home_url( '/sermon-series/' ) ).$item['term']->slug; ?>">
                    <h2 class="entry-title"><?php echo $item['term']->name; ?></h2>
                </a>
                <?php
                    echo get_the_title().'</br>';
                    echo get_the_date(); 
                ?>
            </div>

            <?php
        }

        echo '</div>';  

    }

    echo '</section>';


    echo paginate_links( array (
        'total'   => $max_num_pages,
        'current' => $current_page,
    ) );