Directline Speech - Web shockets error with adapters

Open MunozVictor opened this issue 11 months ago • 0 comments

Now I have managed to make it work, but not through Direct Line Speech:

If I set it to "hybrid" mode, the browser's speech-to-text (STT) works and cognitive service (TTS).
If I set it to "azure" mode, both STT and text-to-speech (TTS) work using the cognitive service.
In "browser" mode, it works using the browser's speech services.

Code that Works Without Direct Line Speech

This is my working implementation without Direct Line Speech, only using Web Speech and Azure Speech Services:

(async function () {
  try {
    // 🔵 1️⃣ IN BACKEND
    const directLineRes = await fetch('https://directline.botframework.com/v3/directline/tokens/generate', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ` + SECRET_DE_DIRECTLINE,
        'Content-Type': 'application/json'
      }
    });
    if (!directLineRes.ok) {
      throw new Error("❌ Error al generar el token de Direct Line.");
    }
    const { token } = await directLineRes.json();
    console.log("✅ Token de Direct Line generado:", token);


	//IN BACKEND
	async function fetchSpeechCredentials() {
	  const res = await fetch("https://northeurope.api.cognitive.microsoft.com/sts/v1.0/issuetoken", {
		method: "POST",
		headers: {
		  "Content-Type": "application/x-www-form-urlencoded",
		  "Content-Length": "0",
		  "Ocp-Apim-Subscription-Key": CLAVE_DE_SPEECH
		}
	  });

	  if (!res.ok) {
		throw new Error("❌ Error al obtener el token de Speech.");
	  }

	  const authorizationToken = await res.text();
	  console.log("✅ Token de Speech obtenido correctamente.");
	  console.log("✅ Token de Speech:"+authorizationToken);

	  return { authorizationToken, region: "northeurope" };
	}
		
		
	async function createPonyfillFactory({ credentials, mode = "hybrid" }) {

    const speechServicesPonyfillFactory = await window.WebChat.createCognitiveServicesSpeechServicesPonyfillFactory({
        credentials
    });

    const webSpeechPonyfillFactory = await window.WebChat.createBrowserWebSpeechPonyfillFactory();

    return options => {
        // Obtener los ponyfills de cada servicio
        const speechServicesPonyfill = speechServicesPonyfillFactory(options);
        const webSpeechPonyfill = webSpeechPonyfillFactory(options);

        if (mode === "azure") {
            console.log("🟢 Modo Azure: Usando Cognitive Services Speech");
            return {
                SpeechGrammarList: speechServicesPonyfill.SpeechGrammarList,
                SpeechRecognition: speechServicesPonyfill.SpeechRecognition,
                speechSynthesis: speechServicesPonyfill.speechSynthesis, //HABILITAR LECTURA DE MENSAJES
              //speechSynthesis: null, //DESHABILITAR LECTURA DE MENSAJES
                SpeechSynthesisUtterance: speechServicesPonyfill.SpeechSynthesisUtterance
            };
        } else if (mode === "browser") {
            console.log("🟠 Modo Navegador: Usando Web Speech API");
            return {
                SpeechGrammarList: webSpeechPonyfill.SpeechGrammarList,
                SpeechRecognition: webSpeechPonyfill.SpeechRecognition,
                speechSynthesis: webSpeechPonyfill.speechSynthesis,
                SpeechSynthesisUtterance: webSpeechPonyfill.SpeechSynthesisUtterance
            };
        } else {
            console.log("🔵 Modo Híbrido: Speech de Azure + Síntesis del navegador");
            return {
                SpeechGrammarList: webSpeechPonyfill.SpeechGrammarList,
                SpeechRecognition: webSpeechPonyfill.SpeechRecognition,
                speechSynthesis: speechServicesPonyfill.speechSynthesis, // Usa síntesis del navegador
                SpeechSynthesisUtterance: speechServicesPonyfill.SpeechSynthesisUtterance
            };
        }
    };
	}

    const webSpeechPonyfillFactory = await window.WebChat.createCognitiveServicesSpeechServicesPonyfillFactory({
      credentials: await fetchSpeechCredentials
    });

    const directLine = window.WebChat.createDirectLine({ token });

    const store = window.WebChat.createStore({}, ({ dispatch }) => next => action => {
      console.log("➡ Acción recibida:", action.type);

      if (action.type === 'DIRECT_LINE/CONNECT_FULFILLED') {
        dispatch({
          type: 'WEB_CHAT/SEND_EVENT',
          payload: {
            name: 'webchat/join',
            value: {
              language: "es-ES",
              mail: "[email protected]",
              client: "web",
              centro: "ss",
              ambito: "logado",
              traces: "no"
            }
          }
        });
      }
	  if (action.type === 'DIRECT_LINE/INCOMING_ACTIVITY') {	
			const activity = action.payload.activity;
			// Verifica si el mensaje es del bot y tiene texto para sintetizar
			if (activity.from.role === 'bot' && activity.type === 'message') {
			  console.log("📢 Mensaje del bot recibido:", activity.text);
			}
		}
		return next(action);
	});


    window.WebChat.renderWebChat({
      directLine,
      //webSpeechPonyfillFactory,  // Usa Cognitive Services Speech para voz
	  webSpeechPonyfillFactory: await createPonyfillFactory({ credentials: await fetchSpeechCredentials , mode:"azure"}),
      store
    }, document.getElementById('webchat'));

    console.log("✅ Web Chat renderizado correctamente.");
    document.querySelector('#webchat > *').focus();
  } catch (error) {
    console.error("❌ Error en la inicialización:", error);
  }
})();

Issue with Direct Line Speech

This behavior might seem correct at first glance, but in reality, you can run this configuration without enabling Direct Line Speech in the Azure Channel, and it will still work in "azure" mode. In other words, Direct Line Speech is doing nothing.

I found this documentation that seems to properly configure Direct Line Speech: [Direct Line Speech Setup](https://github.com/microsoft/BotFramework-WebChat/blob/main/docs/DIRECT_LINE_SPEECH.md#render-web-chat-using-direct-line-speech-adapters)

However, it does not explicitly mention how to retrieve the credentials. From my understanding, we should generate a token for Cognitive Services Speech just like in my previous implementation and then use that token to establish the Direct Line Speech connection.

Following the example from the official documentation (using Direct Line Speech adapters), I cannot get it to work.

** Attempt to Use Direct Line Speech (Not Working)

Following the documentation's example, I tried implementing Direct Line Speech Adapters like this:

	async function fetchCredentials() {
	  const res = await fetch("https://northeurope.api.cognitive.microsoft.com/sts/v1.0/issuetoken", {
		method: "POST",
		headers: {
		  "Content-Type": "application/x-www-form-urlencoded",
		  "Content-Length": "0",
		  "Ocp-Apim-Subscription-Key": CLAVE_DE_SPEECH
		}
	  });

	  if (!res.ok) {
		throw new Error("❌ Error al obtener el token de Speech.");
	  }

	  const authorizationToken = await res.text();

	  return { authorizationToken, region: "northeurope" };
	}
 

	const adapters = await window.WebChat.createDirectLineSpeechAdapters({
	  fetchCredentials
	});

	}

    // Crear la store para manejar eventos y enviar 'webchat/join'
    const store = window.WebChat.createStore({}, ({ dispatch }) => next => action => {
      console.log("➡ Acción recibida:", action.type);
  
      if (action.type === 'DIRECT_LINE/CONNECT_FULFILLED') {
        console.log("✅ Conectado a Direct Line. Enviando evento 'webchat/join'...");
        dispatch({
          type: 'WEB_CHAT/SEND_EVENT',
          payload: {
            name: 'webchat/join',
            value: {
              language: "es-ES",
              mail: "[email protected]",
              client: "web",
              centro: "ss",
              ambito: "logado",
              traces: "no"
            }
          }
        });
      }
  
      if (action.type === 'DIRECT_LINE/INCOMING_ACTIVITY') {
        const activity = action.payload.activity;
        if (activity.from.role === 'bot' && activity.type === 'message') {
          console.log("📢 Mensaje del bot recibido:", activity.text);
        }
      }
      return next(action);
    });
  
    window.WebChat.renderWebChat(
      {
        ...adapters,  
        store
      },
      document.getElementById('webchat')
    );
  
    console.log("✅ Web Chat renderizado correctamente.");
    document.querySelector('#webchat > *').focus();
  } catch (error) {
    console.error("❌ Error en la inicialización:", error);
  }
})();