Delete folder before HttpResponseMessage return

I want to delete a folder and its content before returning some data from the browser.

I am using ASP .NET MVC 5 and just javascript with some DevExpress components.

The method should return a HttpResponseMessage, which is the excel file. The file is created in a temp folder and after that I want to delete it.

I get always an "UnauthorizedAccessException" exception.

Any idea how the logic flow should be to return the file and delete it? I was thinking on calling a new method after finishing the ExportExcelChart but maybe there is a better way to achieve the expected result.

How

JS (browser)

function SaveChart(s, e) {
    //debugger;
    if (e.item.name === 'mnuSaveToDisk') {
        averageChart.printOptions.landscape = true;
        averageChart.printOptions.SetSizeMode('Stretch');
        averageChart.SaveToDisk('pdf', "average curves");
    } else if (e.item.name == 'mnExportChart') {
        ChartLoadingPanel.Show();
        @*$.get('@Url.Action("ExportExcelChart", "Mindboard")', {}, null);*@
        window.location = '@Url.Action("ExportExcelChart", "Mindboard")';
        ChartLoadingPanel.Hide();
    }
}

Controller

public HttpResponseMessage ExportExcelChart()
{
    try
    {
        // create excel file
        String tmpRandomFolderName = DataService.RandomString(20);
        string tmpFolderPath = Path.Combine(Server.MapPath("~/App_Data"), "TEMP", tmpRandomFolderName);
        string tmpOriginFile = Server.MapPath("~/App_Data") + "/ExportChartAverage.xlsx";
        string tmpNewFile = tmpFolderPath + "/ExportChartAverage.xlsx";

        if (!Directory.Exists(tmpFolderPath))
            Directory.CreateDirectory(tmpFolderPath);

        System.IO.File.Copy(tmpOriginFile, tmpNewFile);

        // open excel file
        FileInfo tmpExcelFile = new FileInfo(tmpNewFile);
        ExcelPackage pck = new ExcelPackage(tmpExcelFile);
        var ws = pck.Workbook.Worksheets[1];

        // do stuff...

        pck.SaveAs(Response.OutputStream);
        Response.ContentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
        Response.AddHeader("content-disposition", "attachment;  filename=ExportChartAverage.xlsx");

        if (Directory.Exists(tmpFolderPath))
        {
            if (System.IO.File.Exists(tmpNewFile))
                System.IO.File.Delete(tmpNewFile);
            Directory.Delete(tmpFolderPath, true);
        }

        var response = new HttpResponseMessage(HttpStatusCode.OK);
        return response;
    }
    catch (IOException ex)
    {
        var response = new HttpResponseMessage(HttpStatusCode.NotFound);
        response.Content = new StringContent(ex.Message);

        return response;
        //throw ex;
    }
    catch (UnauthorizedAccessException ex)
    {
        var response = new HttpResponseMessage(HttpStatusCode.NotFound);
        response.Content = new StringContent(ex.Message);

        return response;
        //throw ex;
    }
    catch (Exception)
    {

        throw;
    }
}