Example using ws2tcpip troubles

Windows specific questions.
Post Reply
UEZ
Posts: 972
Joined: May 05, 2017 19:59
Location: Germany

Example using ws2tcpip troubles

Post by UEZ »

I want to have an alternative to WinHTTP API to read out the source code of a web site. I tried to convert a C++ example from here to FB but it doesn't work properly. The reply from server is

Code: Select all

HTTP/1.1 301 Moved Permanently
Server: nginx/1.18.0 (Ubuntu)
Date: Thu, 18 Mar 2021 13:58:56 GMT
Content-Type: text/html
Content-Length: 178
Connection: close
Location: https://remix.kwed.org/
...
Here the FB code:

Code: Select all

'#Include "win\winsock2.bi"
#Include "win\ws2tcpip.bi"
'#Inclib "ws2_32.lib"

Dim As String url = "remix.kwed.org"

'Initialize Dependencies to the Windows Socket.
Dim As WSADATA wsaData
If WSAStartup(MAKEWORD(2, 2), @wsaData) Then
	? "WSAStartup failed"
	End -1
End If

'We first prepare some "hints" for the "getaddrinfo" function
'to tell it, that we are looking for a IPv4 TCP Connection.
Dim As ADDRINFO hints
hints.ai_family = AF_INET   		'targeting IPv4
hints.ai_protocol = IPPROTO_TCP		'targeting TCP
hints.ai_socktype = SOCK_STREAM		'targeting TCP so its SOCK_STREAM

'Aquiring of the IPv4 address of a host using the newer
'"getaddrinfo" function which outdated "gethostbyname".
'It will search for IPv4 addresses using the TCP-Protocol.
Dim As ADDRINFO Ptr targetAdressInfo
Dim As DWORD getAddrRes
getAddrRes = getaddrinfo(Strptr(url), 0, @hints, @targetAdressInfo) 'https://docs.microsoft.com/en-us/windows/win32/api/ws2tcpip/nf-ws2tcpip-getaddrinfo
If getAddrRes <> 0 Or targetAdressInfo = 0 Then
	? "Could not resolve the Host Name"
	WSACleanup()
	End -2
End If

'Create the Socket Address Informations, using IPv4
'We dont have to take care of sin_zero, it is only used to extend the length of SOCKADDR_IN to the size of SOCKADDR
Dim As SOCKADDR_IN sockAddr
sockAddr.sin_addr = Cptr(sockaddr_in Ptr, targetAdressInfo->ai_addr)->sin_addr
sockAddr.sin_family = AF_INET
sockAddr.sin_port = htons(80)

'We have to free the Address-Information from getaddrinfo again
freeaddrinfo(targetAdressInfo)

'Creation of a socket for the communication with the Web Server,
'using IPv4 and the TCP-Protocol
Dim As SOCKET webSocket = socket_(AF_INET, SOCK_STREAM, IPPROTO_TCP)
If (webSocket = INVALID_SOCKET) Then
	? "Creation of the Socket Failed"
	WSACleanup()
	End -3	
End If

'Establishing a connection to the web Socket
? "Connecting...";
If connect(webSocket, Cptr(SOCKADDR Ptr, @sockAddr), Sizeof(SOCKADDR_IN)) <> 0 Then
	? "Could not connect"
	closesocket(webSocket)
	WSACleanup()
	End -3	
End If
? "Connected"

#Define CRLF Chr(13, 10) '!"\r\n"
'Sending a HTTP-GET-Request to the Web Server "GET / HTTP/1.1\r\nHost: www.google.com\r\nConnection: close\r\n\r\n";
Dim As String httpRequest = "GET / HTTP/1.1" & CRLF & "Host: " & url & CRLF & "Accept: text/html;text/plain" & CRLF & "Connection: close" & CRLF & CRLF
Dim As Integer sentBytes = send(webSocket, Strptr(httpRequest), Len(httpRequest), 0)
If sentBytes < Len(httpRequest) Or sentBytes = SOCKET_ERROR Then
	? "Could not send the request to the Server"
	closesocket(webSocket)
	WSACleanup()	
	End -4
End If

'Receiving and Displaying an answer from the Web Server
Dim As Ubyte buffer(500000)
Dim As Integer dataLen, i
Dim As String website_HTML

While True
	dataLen = recv(webSocket, @buffer(0), Ubound(buffer), 0)
	If dataLen = 0 Then Exit While
	i = 0
	While (buffer(i) > 31) Or (buffer(i) = 13) Or (buffer(i) = 10)
		website_HTML &= Chr(buffer(i))
		i += 1
	Wend
Wend

'Cleaning up Windows Socket Dependencies
closesocket(webSocket)
WSACleanup()
?
? website_HTML
?
? "Press any key to exit"
Sleep
Do you know what's wrong?

Thx.
D.J.Peters
Posts: 8586
Joined: May 28, 2005 3:28
Contact:

Re: Example using ws2tcpip troubles

Post by D.J.Peters »

Why not using SNC imple [N]etwork [C]onnection works on Linux also

Joshy

Code: Select all

' [S]imple [N]etwork [C]onnection download a web size
#include once "snc.bi"
#include once "snc_utility.bi"

' test of a client connection 
const as string ServerName = "shiny3d.de"
const as string ServerPath = "/public/"
const as string ServerFile = "index.html"
const as string FileType   = MIME_HTM & ";" & MIME_TXT

' connect to web server at port 80
dim as NetworkClient client=type(ServerName,80)
' get a connection from ConnectionFactory
var connection = client.GetConnection()
' build an HTTP GET request
dim as string request =HTTPGet(ServerName,ServerPath & ServerFile,FileType)

' ready to send ?
while connection->CanPut()<>1
  sleep 100
wend
' put data on the connection
connection->PutData(strptr(request),len(request))
' ready to receive ?
while connection->CanGet()<>1
  sleep 100
wend

dim as zstring ptr buffer
var nBytes = connection->GetData(buffer)
print "number of received bytes " & nBytes

' get last char position of the HTTP ascii header
var LastChar=instr(*buffer,HeaderEnd)-1
' get the header from response
var Header = left(*buffer,LastChar)

' is it a OK answer ?
if instr(Header,"200 ")<1 andalso instr(Header,"302 ")<1 then
  deallocate buffer
  print "error: can't get " & ServerName & ServerPath & ServerFile & " !"
  beep : sleep : end 1
end if

' get first byte after header
var DataStart=LastChar+len(HeaderEnd)
' save it
chdir exepath()
var hFile = FreeFile()
if open("page.html",for binary,access write,as hFile) then
  deallocate buffer
  print "error: can't save 'page.html' !"
  beep :sleep :end 1
end if

dim as ubyte ptr FileBuffer=@buffer[DataStart]
nBytes -= DataStart
put #hFile,,*FileBuffer,nBytes
close hFile
print "file 'page.html' saved ..."
' free the buffer (allocate by snc.bi)
deallocate buffer
print "done ..."
sleep
dodicat
Posts: 7976
Joined: Jan 10, 2006 20:30
Location: Scotland

Re: Example using ws2tcpip troubles

Post by dodicat »

Copy and paste a page to get .txt and .html.
(The default page is index.php)

Code: Select all



dim as string g
g += "Const TriStateTrue = -1 ' forUnicode support"+chr(10)
g += "URL = InputBox(""Enter (or paste) the URL to extract the Code ""&vbcr&vbcr&_"+chr(10)
g += """Exemple """"https://www.freebasic.net"""""",""Extraction of Source text and html  "",""https://www.freebasic.net/forum/index.php"")"+chr(10)
g += "If URL = """" Then WScript.Quit"+chr(10)
g += "Titre = ""Extraction du Code Source de "" & URL"+chr(10)
g += "Set ie = CreateObject(""InternetExplorer.Application"")"+chr(10)
g += "Set objFSO = CreateObject(""Scripting.FileSystemObject"")"+chr(10)
g += "ie.Navigate(URL)"+chr(10)
g += "ie.Visible=false"+chr(10)
g += "DO WHILE ie.busy"+chr(10)
g += "LOOP"+chr(10)
g += "DataHTML = ie.document.documentElement.innerHTML"+chr(10)
g += "DataTxt = ie.document.documentElement.innerText"+chr(10)
g += "strFileHTML = ""CodeSourceHTML.txt"""+chr(10)
g += "strFileTxt = ""CodeSourceTxt.txt"""+chr(10)
g += "Set objHTMLFile = objFSO.OpenTextFile(strFileHTML,2,True, TriStateTrue)"+chr(10)
g += "objHTMLFile.WriteLine(DataHTML)"+chr(10)
g += "objHTMLFile.Close"+chr(10)
g += "Set objTxtFile = objFSO.OpenTextFile(strFileTxt,2,True, TriStateTrue)"+chr(10)
g += "objTxtFile.WriteLine(DataTxt)"+chr(10)
g += "objTxtFile.Close"+chr(10)
g += "ie.Quit"+chr(10)
g += "Set ie=Nothing"+chr(10)
g += " Ouvrir(strFileHTML)"+chr(10)
g += " Ouvrir(strFileTxt)"+chr(10)
g += "wscript.Quit"+chr(10)
g += "Function Ouvrir(File)"+chr(10)
g += "    Set ws=CreateObject(""wscript.shell"")"+chr(10)
g += "    ws.run ""Notepad.exe ""& File,1,False"+chr(10)
g += "end Function"+chr(10)



function savefile(filename As String,p As String) as string
    Dim As long n=freefile
    If Open (filename For Binary Access Write As #n)=0 Then
        Put #n,,p
        Close
    Else
        Print "Unable to save " + filename:sleep:end
    End If
    return filename
End function

Sub runscript(filename As String) 
  Shell "cscript.exe /Nologo "+ filename 
End Sub


savefile("script.vbs",g)
runscript("script.vbs")
kill "script.vbs"
sleep
 
jj2007
Posts: 2326
Joined: Oct 23, 2016 15:28
Location: Roma, Italia
Contact:

Re: Example using ws2tcpip troubles

Post by jj2007 »

Print NoTag$(FileRead$("http://remix.kwed.org")) works fine. WinInet under the hood.

Code: Select all

Remix.Kwed.Org | The Def Guide to C64 MP3 remakes
C64 Take-away Episode 056 released on 06/12-2019
Tune into the C64 Take-away Podcast ( RSS Feed ) and get C64 remixes and SIDs automatically transferred to your desktop, smartphone or portable MP3 player. Recent news
26/02/2021 15:41
Attn. Mark Wright - emails to your r**********ly.co.uk account bounce. Please get in touch with a working e-mail address.
02/01/2021 21:32
Luckily 2020 wasn't all bad. 174 new remixes were released and you can get all of them with a single click using the RKO 2020 torrent . And please help your fellow remix fan by
 seeding as long as you can.
Free MP3s? Nice! Can I use them for anything? Yes and no. You are free to add the files obtained here to your music library and sync them to your MP3 player. But...
UEZ
Posts: 972
Joined: May 05, 2017 19:59
Location: Germany

Re: Example using ws2tcpip troubles

Post by UEZ »

@Joshy: I tested SNC but somehow I didn't get it work to grab the source code of a web site. I know that SNC uses similar function, can you see the bug in the code I posted?
Using your code

Code: Select all

...
' test of a client connection
const as string ServerName = "remix.kwed.org"
const as string ServerPath = "/"
const as string ServerFile = ""
const as string FileType   = MIME_HTM & ";" & MIME_TXT
...
Output:

Code: Select all

number of received bytes 380
error: can't get remix.kwed.org/ !
@dodicat:thanks, that vbs solution is working

@jj2007: eh? How to use that?
jj2007
Posts: 2326
Joined: Oct 23, 2016 15:28
Location: Roma, Italia
Contact:

Re: Example using ws2tcpip troubles

Post by jj2007 »

UEZ wrote:@jj2007: eh? How to use that?
FileRead$(url) uses good ol' WinInet.
adeyblue
Posts: 299
Joined: Nov 07, 2019 20:08

Re: Example using ws2tcpip troubles

Post by adeyblue »

I don't know if you've managed to see the connection, but the reason your first post and SNC return 301 responses and JJs and the VBS work is because the site is redirecting non secure requests to HTTPS. Your code (and presumably SNC too) doesn't handle redirects and/or TLS, while those higher level APIs do.

I'm not sure there is any easy low-level API way to manually do TLS on sockets. The SChannel authentication package is the built-in thing that handles SSL and TLS connections, example on how to use it though aren't particularly plentiful (if that even is the thing to use in this situaton)
UEZ
Posts: 972
Joined: May 05, 2017 19:59
Location: Germany

Re: Example using ws2tcpip troubles

Post by UEZ »

adeyblue wrote:I don't know if you've managed to see the connection, but the reason your first post and SNC return 301 responses and JJs and the VBS work is because the site is redirecting non secure requests to HTTPS. Your code (and presumably SNC too) doesn't handle redirects and/or TLS, while those higher level APIs do.

I'm not sure there is any easy low-level API way to manually do TLS on sockets. The SChannel authentication package is the built-in thing that handles SSL and TLS connections, example on how to use it though aren't particularly plentiful (if that even is the thing to use in this situaton)
That makes sense. I'm using in general WinHTTP API which supports TLS on Win8+. Win7 needs some updates to support also TLS.
UEZ
Posts: 972
Joined: May 05, 2017 19:59
Location: Germany

Re: Example using ws2tcpip troubles

Post by UEZ »

dodicat wrote:Copy and paste a page to get .txt and .html.
(The default page is index.php)

Code: Select all



dim as string g
g += "Const TriStateTrue = -1 ' forUnicode support"+chr(10)
g += "URL = InputBox(""Enter (or paste) the URL to extract the Code ""&vbcr&vbcr&_"+chr(10)
g += """Exemple """"https://www.freebasic.net"""""",""Extraction of Source text and html  "",""https://www.freebasic.net/forum/index.php"")"+chr(10)
g += "If URL = """" Then WScript.Quit"+chr(10)
g += "Titre = ""Extraction du Code Source de "" & URL"+chr(10)
g += "Set ie = CreateObject(""InternetExplorer.Application"")"+chr(10)
g += "Set objFSO = CreateObject(""Scripting.FileSystemObject"")"+chr(10)
g += "ie.Navigate(URL)"+chr(10)
g += "ie.Visible=false"+chr(10)
g += "DO WHILE ie.busy"+chr(10)
g += "LOOP"+chr(10)
g += "DataHTML = ie.document.documentElement.innerHTML"+chr(10)
g += "DataTxt = ie.document.documentElement.innerText"+chr(10)
g += "strFileHTML = ""CodeSourceHTML.txt"""+chr(10)
g += "strFileTxt = ""CodeSourceTxt.txt"""+chr(10)
g += "Set objHTMLFile = objFSO.OpenTextFile(strFileHTML,2,True, TriStateTrue)"+chr(10)
g += "objHTMLFile.WriteLine(DataHTML)"+chr(10)
g += "objHTMLFile.Close"+chr(10)
g += "Set objTxtFile = objFSO.OpenTextFile(strFileTxt,2,True, TriStateTrue)"+chr(10)
g += "objTxtFile.WriteLine(DataTxt)"+chr(10)
g += "objTxtFile.Close"+chr(10)
g += "ie.Quit"+chr(10)
g += "Set ie=Nothing"+chr(10)
g += " Ouvrir(strFileHTML)"+chr(10)
g += " Ouvrir(strFileTxt)"+chr(10)
g += "wscript.Quit"+chr(10)
g += "Function Ouvrir(File)"+chr(10)
g += "    Set ws=CreateObject(""wscript.shell"")"+chr(10)
g += "    ws.run ""Notepad.exe ""& File,1,False"+chr(10)
g += "end Function"+chr(10)



function savefile(filename As String,p As String) as string
    Dim As long n=freefile
    If Open (filename For Binary Access Write As #n)=0 Then
        Put #n,,p
        Close
    Else
        Print "Unable to save " + filename:sleep:end
    End If
    return filename
End function

Sub runscript(filename As String) 
  Shell "cscript.exe /Nologo "+ filename 
End Sub


savefile("script.vbs",g)
runscript("script.vbs")
kill "script.vbs"
sleep
 
Just a remark: I converted the VBS code to FB but it only runs properly when "Enable Protected Mode" in IE security setting for Internet is unchecked otherwise error messages will appear. But why does VBS work without the modification?
dodicat
Posts: 7976
Joined: Jan 10, 2006 20:30
Location: Scotland

Re: Example using ws2tcpip troubles

Post by dodicat »

I have had these problems also.
disphelper works OK also.

Code: Select all

#define UNICODE
#include "windows.bi"
#include once "disphelper/disphelper.bi"
type Chars
    FBChars as IDispatch ptr
    declare function Text() as string
    end type
type doc
    FBdoc as IDispatch ptr
    declare function TextLines() as Chars
    end type
type IE
    FBIE as IDispatch Ptr
    declare function MainDoc() as doc
    declare sub FindPage( as string)
    declare property Show (as integer)
    declare sub Start()
    declare constructor()
    declare destructor()
    declare operator cast() as string
end type

constructor IE
Start
this.Show = true
end constructor

destructor IE
dhcallmethod( FBIE ,"quit")
end destructor

operator IE.cast() as string
var s= this.MainDoc.TextLines.Text
return s+chr(10)+"done"
end operator

sub IE.Start()
    dhInitialize(TRUE)
    dhToggleExceptions(FALSE)
    dhCreateObject("InternetExplorer.Application",NULL,@FBIE)
end sub

property IE.Show( I as integer )
    dhPutValue(FBIE, ".Visible = %b", I)
end property

sub IE.FindPage(S as string)
    dhcallmethod(FBIE, ".Navigate %s",S)
end sub

function IE.MainDoc() as doc
    dim temp as doc
        dhGetValue("%o", @temp.FBdoc,FBIE, "Document")
    return temp
end function

function doc.TextLines() as Chars
    dim temp as Chars
        dhGetValue("%o", @temp.FBChars, FBdoc, "Body")
    return temp
end function

function Chars.Text() as string
    Dim temp As Zstring Ptr
        dhGetValue("%s", @temp, FBChars, "innerHTML")''<=== or innerText
    return *temp
end function

'========================== USAGE  =======================================

dim as IE mypage

mypage.FindPage "https://forum.qbasic.at/viewtopic.php?t=8996"

print "TEXT PAGE, Make active and press a key when ready":print
sleep
print mypage.MainDoc.TextLines.Text
sleep
'==========  OPTIONAL ====================
open "internetdata.txt" for output as #1
print #1,mypage
close #1

shell "notepad internetdata.txt"
print "Press a key to exit"
sleep

  
You need libdisphelper.a of course.
Post Reply