interrupt functions need to be as short as possible, run from RAM and everything in it (e.g. every function you call from within) also needs to run from RAM. In your case only onTimer() is in RAM, but everything else is probably not. All this is causing the watchdog to trigger.

To solve the issue, you should only set a flag in the interrupt function and test this flag in the loop() function and then act upon it.

Try something like this:

#include <M5StickC.h>  //define M5StickC hardware components and functions
#define BUTTON_MAIN 37  //The main button pin number
#define SCREEN_TIMEOUT 10 //Screen-off timeout in seconds
hw_timer_t *timer;

volatile bool onButtonFlag = false;
volatile bool onTimerFlag = false;

void IRAM_ATTR onButton() { //Interrupt on button press => turn on the screen
  onButtonFlag = true;
}

void IRAM_ATTR onTimer(){  //Interrupt on timer => turn off the screen
  onTimerFlag = true;
}

void setup() {
M5.begin(); //Init M5
M5.Lcd.fillScreen(YELLOW); //Make screen yellow
pinMode(BUTTON_MAIN, INPUT_PULLUP); //Button GPIO set as input, pull-up enabled
timer=timerBegin(1,40000,true); //Init screen light timer, divisor 40000 => 0.5ms steps
timerAttachInterrupt(timer,&onTimer,true);  //Set screen light timer interrupt function
timerAlarmWrite(timer,SCREEN_TIMEOUT*2000,true); //Set interrupt to turn off screen after X timer steps of 0.5ms each.
timerAlarmEnable(timer); //Start timer for screen-off delay
}

void loop() {
  if(onButtonFlag) {
    onButtonFlag = false;
    detachInterrupt(BUTTON_MAIN); //Ignore more button presses
    M5.Axp.ScreenBreath(10); //Set screen light ON
    timerAlarmEnable(timer); //Start timer for screen-off delay
  }
  if(onTimerFlag) {
    onTimerFlag = false;
    timerAlarmDisable(timer); //Stop the timer.
    M5.Axp.ScreenBreath(0); //Set screen light OFF
    attachInterrupt(BUTTON_MAIN,onButton,FALLING); //Attach interrupt on button press => turn on the screen
  }
}

Cheers
Felix