Payment field disappears on custom Paypal plugin

I have a custom Paypal plugin.

I get the Paypal button to appear under the payment method box in checkout but after ?wc-ajax=get_refreshed_fragments is ran, initiated by vendors~rollbar.noconflict.umd.min.js, the button disappears, I have no error in my console.

On my payment_fields() function I have a single div that is populated with the paypal button from the paypal SDK script like this:

 function payment_fields(){
      ?> <div id="paypal-button-container"></div><?php
  }

This is how I enqueue my scripts:

function enqueueScripts(){
     if(is_checkout()){
   
     $currency        = get_woocommerce_currency();
     $client_id       = 'test';
     $paypal_endpoint = 'https://www.paypal.com/sdk/js';

     $arg = array(
         'currency'  => $currency,
         'client-id' => $client_id,
     );
     $paypal_src      = add_query_arg($arg, $paypal_endpoint.'#defer');

     wp_enqueue_script( 'paypal-sdk', $paypal_src, array(), null);
     wp_enqueue_script( 'paypal-button', plugin_dir_url(__FILE__) . '../js/button.min.js#defer', array('jquery'));
 }


 }

The script to populate the div with the paypal button, paypal-button.js:

jQuery(document).ready(function($) {
paypal.Buttons({

    // Sets up the transaction when a payment button is clicked
    createOrder: function(data, actions) {
        return actions.order.create({
            purchase_units: [{
                amount: {
                    value: '77.44' // Can reference variables or functions. Example: `value: document.getElementById('...').value`
                }
            }]
        });
    },

    // Finalize the transaction after payer approval
    onApprove: function(data, actions) {
        return actions.order.capture().then(function(orderData) {
            // Successful capture! For dev/demo purposes:
            console.log('Capture result', orderData, JSON.stringify(orderData, null, 2));
            var transaction = orderData.purchase_units[0].payments.captures[0];
            alert('Transaction '+ transaction.status + ': ' + transaction.id + 'nnSee console for all available details');

            // When ready to go live, remove the alert and show a success message within this page. For example:
            // var element = document.getElementById('paypal-button-container');
            // element.innerHTML = '';
            // element.innerHTML = '<h3>Thank you for your payment!</h3>';
            // Or go to another URL:  actions.redirect('thank_you.html');
        });
    }
}).render('#paypal-button-container');
}); 

Scripts are loaded through this function:

function loadFiltersActions(){
   add_action( 'wp_enqueue_scripts', array($this, 'enqueueScripts'));
     if(is_admin()) {
         add_action( 'woocommerce_update_options_payment_gateways_' . $this->id, array( $this, 'process_admin_options' ) );
       }
}

In my construct I have this:

 function __construct()
 {
 $this->id    = 'test_paypal';
 $this->icon  = '';

 $this->title              = _('Paypal', 'test-paypal');
 $this->method_title       = _('Paypal', 'test-paypal');
 $this->method_description = _('Custom built Paypal Plugin because anything else sucks', 'test-paypal');

 $this->init_form_fields();
 $this->init_settings();

 // load settings
 $this->title         = $this->get_option( 'title' );
 $this->description   = $this->get_option( 'description' );
 $this->enabled       = $this->get_option( 'enabled' );

 $this->loadFiltersActions();


}

So if I’m instead enqueuing the scripts directly in the payment_fields function inline like this:

   function payment_fields(){
  $currency        = get_woocommerce_currency();
  $client_id       = 'test';
  $paypal_endpoint = 'https://www.paypal.com/sdk/js';

  $arg = array(
     'currency'  => $currency,
     'client-id' => $client_id,
  );
  $paypal_src      = add_query_arg($arg, $paypal_endpoint);
  ?> <div id="paypal-button-container"></div><?php

  ?>
  <script src="<?php echo $paypal_src; ?>" defer="defer"></script>
  <script src="<?php echo plugin_dir_url(__FILE__) . '../js/button.js' ;?>" defer="defer"></script>

  <?php


}

I get this error on my console: